{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Deep LSTMs with Word2vec using RNN API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import collections\n",
    "import math\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import nltk\n",
    "import zipfile\n",
    "from matplotlib import pylab\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "import tensorflow as tf\n",
    "from tensorflow.contrib import rnn\n",
    "import csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading Stories\n",
    "Stories are automatically downloaded from https://www.cs.cmu.edu/~spok/grimmtmp/, if not detected in the disk. The total size of stories is around ~500KB. The dataset consists of 100 stories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading file:  stories\\001.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\002.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\003.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\004.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\005.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\006.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\007.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\008.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\009.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\010.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\011.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\012.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\013.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\014.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\015.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\016.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\017.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\018.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\019.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\020.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\021.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\022.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\023.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\024.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\025.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\026.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\027.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\028.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\029.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\030.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\031.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\032.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\033.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\034.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\035.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\036.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\037.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\038.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\039.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\040.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\041.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\042.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\043.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\044.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\045.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\046.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\047.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\048.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\049.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\050.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\051.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\052.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\053.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\054.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\055.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\056.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\057.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\058.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\059.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\060.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\061.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\062.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\063.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\064.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\065.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\066.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\067.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\068.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\069.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\070.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\071.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\072.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\073.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\074.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\075.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\076.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\077.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\078.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\079.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\080.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\081.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\082.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\083.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\084.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\085.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\086.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\087.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\088.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\089.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\090.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\091.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\092.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\093.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\094.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\095.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\096.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\097.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\098.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\099.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\100.txt\n",
      "Not downloading. File already exists.\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.cs.cmu.edu/~spok/grimmtmp/'\n",
    "\n",
    "# Create a directory if needed\n",
    "dir_name = 'stories'\n",
    "if not os.path.exists(dir_name):\n",
    "    os.mkdir(dir_name)\n",
    "    \n",
    "def maybe_download(filename):\n",
    "  \"\"\"Download a file if not present\"\"\"\n",
    "  print('Downloading file: ', dir_name+ os.sep+filename)\n",
    "    \n",
    "  if not os.path.exists(dir_name+os.sep+filename):\n",
    "    filename, _ = urlretrieve(url + filename, dir_name+os.sep+filename)\n",
    "  else:\n",
    "    print('Not downloading. File already exists.')\n",
    "  statinfo = os.stat(dir_name+os.sep+filename)\n",
    "  \n",
    "  return filename\n",
    "\n",
    "num_files = 100\n",
    "filenames = [format(i, '03d')+'.txt' for i in range(1,num_files+1)]\n",
    "\n",
    "for fn in filenames:\n",
    "    maybe_download(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 files found.\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(filenames)):\n",
    "    file_exists = os.path.isfile(os.path.join(dir_name,filenames[i]))\n",
    "    assert file_exists\n",
    "print('%d files found.'%len(filenames))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading data\n",
    "Data will be stored in a list of lists where the each list represents a document and document is a list of words. We will then break the text into words."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file stories\\001.txt\n",
      "Data size (Characters) (Document 0) 1693\n",
      "Sample string (Document 0) ['in', 'olden', 'times', 'when', 'wishing', 'still', 'helped', 'one', ',', 'there', 'lived', 'a', 'king', 'whose', 'daughters', 'were', 'all', 'beautiful', ',', 'but', 'the', 'youngest', 'was', 'so', 'beautiful', 'that', 'the', 'sun', 'itself', ',', 'which', 'has', 'seen', 'so', 'much', ',', 'was', 'astonished', 'whenever', 'it', 'shone', 'in', 'her', 'face', '.', 'close', 'by', 'the', 'king', \"'s\"]\n",
      "\n",
      "Processing file stories\\002.txt\n",
      "Data size (Characters) (Document 1) 2167\n",
      "Sample string (Document 1) ['hard', 'by', 'a', 'great', 'forest', 'dwelt', 'a', 'wood-cutter', 'with', 'his', 'wife', ',', 'who', 'had', 'an', 'only', 'child', ',', 'a', 'little', 'girl', 'three', 'years', 'old', '.', 'they', 'were', 'so', 'poor', ',', 'however', ',', 'that', 'they', 'no', 'longer', 'had', 'daily', 'bread', ',', 'and', 'did', 'not', 'know', 'how', 'to', 'get', 'food', 'for', 'her']\n",
      "\n",
      "Processing file stories\\003.txt\n",
      "Data size (Characters) (Document 2) 4499\n",
      "Sample string (Document 2) ['a', 'certain', 'father', 'had', 'two', 'sons', ',', 'the', 'elder', 'of', 'whom', 'was', 'smart', 'and', 'sensible', ',', 'and', 'could', 'do', 'everything', ',', 'but', 'the', 'younger', 'was', 'stupid', 'and', 'could', 'neither', 'learn', 'nor', 'understand', 'anything', ',', 'and', 'when', 'people', 'saw', 'him', 'they', 'said', \"'there\", \"'s\", 'a', 'fellow', 'who', 'will', 'give', 'his', 'father']\n",
      "\n",
      "Processing file stories\\004.txt\n",
      "Data size (Characters) (Document 3) 1265\n",
      "Sample string (Document 3) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'goat', 'who', 'had', 'seven', 'little', 'kids', ',', 'and', 'loved', 'them', 'with', 'all', 'the', 'love', 'of', 'a', 'mother', 'for', 'her', 'children', '.', 'one', 'day', 'she', 'wanted', 'to', 'go', 'into', 'the', 'forest', 'and', 'fetch', 'some', 'food', '.', 'so', 'she', 'called', 'all', 'seven', 'to', 'her']\n",
      "\n",
      "Processing file stories\\005.txt\n",
      "Data size (Characters) (Document 4) 3590\n",
      "Sample string (Document 4) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'king', 'who', 'was', 'ill', 'and', 'thought', 'to', 'himself', \"'i\", 'am', 'lying', 'on', 'what', 'must', 'be', 'my', 'deathbed', '.', \"'\", 'then', 'said', 'he', \"'tell\", 'faithful', 'john', 'to', 'come', 'to', 'me', '.', \"'\", 'faithful', 'john', 'was', 'his', 'favorite', 'servant', ',', 'and', 'was', 'so', 'called']\n",
      "\n",
      "Processing file stories\\006.txt\n",
      "Data size (Characters) (Document 5) 2005\n",
      "Sample string (Document 5) ['there', 'was', 'once', 'a', 'peasant', 'who', 'had', 'driven', 'his', 'cow', 'to', 'the', 'fair', ',', 'and', 'sold', 'her', 'for', 'seven', 'talers', '.', 'on', 'the', 'way', 'home', 'he', 'had', 'to', 'pass', 'a', 'pond', ',', 'and', 'already', 'from', 'afar', 'he', 'heard', 'the', 'frogs', 'crying', ',', 'aik', ',', 'aik', ',', 'aik', ',', 'aik', '.']\n",
      "\n",
      "Processing file stories\\007.txt\n",
      "Data size (Characters) (Document 6) 2265\n",
      "Sample string (Document 6) ['there', 'were', 'once', 'upon', 'a', 'time', 'a', 'king', 'and', 'a', 'queen', 'who', 'lived', 'happily', 'together', 'and', 'had', 'twelve', 'children', ',', 'but', 'they', 'were', 'all', 'boys', '.', 'then', 'said', 'the', 'king', 'to', 'his', 'wife', ',', 'if', 'the', 'thirteenth', 'child', 'which', 'you', 'are', 'about', 'to', 'bring', 'into', 'the', 'world', ',', 'is', 'a']\n",
      "\n",
      "Processing file stories\\008.txt\n",
      "Data size (Characters) (Document 7) 2781\n",
      "Sample string (Document 7) ['little', 'brother', 'took', 'his', 'little', 'sister', 'by', 'the', 'hand', 'and', 'said', ',', 'since', 'our', 'mother', 'died', 'we', 'have', 'had', 'no', 'happiness', '.', 'our', 'step-mother', 'beats', 'us', 'every', 'day', ',', 'and', 'if', 'we', 'come', 'near', 'her', 'she', 'kicks', 'us', 'away', 'with', 'her', 'foot', '.', 'our', 'meals', 'are', 'the', 'hard', 'crusts', 'of']\n",
      "\n",
      "Processing file stories\\009.txt\n",
      "Data size (Characters) (Document 8) 1635\n",
      "Sample string (Document 8) ['there', 'were', 'once', 'a', 'man', 'and', 'a', 'woman', 'who', 'had', 'long', 'in', 'vain', 'wished', 'for', 'a', 'child', '.', 'at', 'length', 'the', 'woman', 'hoped', 'that', 'god', 'was', 'about', 'to', 'grant', 'her', 'desire', '.', 'these', 'people', 'had', 'a', 'little', 'window', 'at', 'the', 'back', 'of', 'their', 'house', 'from', 'which', 'a', 'splendid', 'garden', 'could']\n",
      "\n",
      "Processing file stories\\010.txt\n",
      "Data size (Characters) (Document 9) 2353\n",
      "Sample string (Document 9) ['there', 'was', 'once', 'a', 'man', 'whose', 'wife', 'died', ',', 'and', 'a', 'woman', 'whose', 'husband', 'died', ',', 'and', 'the', 'man', 'had', 'a', 'daughter', ',', 'and', 'the', 'woman', 'also', 'had', 'a', 'daughter', '.', 'the', 'girls', 'were', 'acquainted', 'with', 'each', 'other', ',', 'and', 'went', 'out', 'walking', 'together', ',', 'and', 'afterwards', 'came', 'to', 'the']\n",
      "\n",
      "Processing file stories\\011.txt\n",
      "Data size (Characters) (Document 10) 1038\n",
      "Sample string (Document 10) ['there', 'was', 'once', 'a', 'girl', 'who', 'was', 'idle', 'and', 'would', 'not', 'spin', ',', 'and', 'let', 'her', 'mother', 'say', 'what', 'she', 'would', ',', 'she', 'could', 'not', 'bring', 'her', 'to', 'it', '.', 'at', 'last', 'the', 'mother', 'was', 'once', 'so', 'overcome', 'with', 'anger', 'and', 'impatience', ',', 'that', 'she', 'beat', 'her', ',', 'at', 'which']\n",
      "\n",
      "Processing file stories\\012.txt\n",
      "Data size (Characters) (Document 11) 3427\n",
      "Sample string (Document 11) ['hard', 'by', 'a', 'great', 'forest', 'dwelt', 'a', 'poor', 'wood-cutter', 'with', 'his', 'wife', 'and', 'his', 'two', 'children', '.', 'the', 'boy', 'was', 'called', 'hansel', 'and', 'the', 'girl', 'gretel', '.', 'he', 'had', 'little', 'to', 'bite', 'and', 'to', 'break', ',', 'and', 'once', 'when', 'great', 'dearth', 'fell', 'on', 'the', 'land', ',', 'he', 'could', 'no', 'longer']\n",
      "\n",
      "Processing file stories\\013.txt\n",
      "Data size (Characters) (Document 12) 1619\n",
      "Sample string (Document 12) ['there', 'was', 'once', 'on', 'a', 'time', 'a', 'poor', 'man', ',', 'who', 'could', 'no', 'longer', 'support', 'his', 'only', 'son', '.', 'then', 'said', 'the', 'son', ',', 'dear', 'father', ',', 'things', 'go', 'so', 'badly', 'with', 'us', 'that', 'i', 'am', 'a', 'burden', 'to', 'you', '.', 'i', 'would', 'rather', 'go', 'away', 'and', 'see', 'how', 'i']\n",
      "\n",
      "Processing file stories\\014.txt\n",
      "Data size (Characters) (Document 13) 1813\n",
      "Sample string (Document 13) ['a', 'long', 'time', 'ago', 'there', 'lived', 'a', 'king', 'who', 'was', 'famed', 'for', 'his', 'wisdom', 'through', 'all', 'the', 'land', '.', 'nothing', 'was', 'hidden', 'from', 'him', ',', 'and', 'it', 'seemed', 'as', 'if', 'news', 'of', 'the', 'most', 'secret', 'things', 'was', 'brought', 'to', 'him', 'through', 'the', 'air', '.', 'but', 'he', 'had', 'a', 'strange', 'custom']\n",
      "\n",
      "Processing file stories\\015.txt\n",
      "Data size (Characters) (Document 14) 3834\n",
      "Sample string (Document 14) ['one', 'summer', \"'s\", 'morning', 'a', 'little', 'tailor', 'was', 'sitting', 'on', 'his', 'table', 'by', 'the', 'window', ',', 'he', 'was', 'in', 'good', 'spirits', ',', 'and', 'sewed', 'with', 'all', 'his', 'might', '.', 'then', 'came', 'a', 'peasant', 'woman', 'down', 'the', 'street', 'crying', ',', 'good', 'jams', ',', 'cheap', '.', 'good', 'jams', ',', 'cheap', '.', 'this']\n",
      "\n",
      "Processing file stories\\016.txt\n",
      "Data size (Characters) (Document 15) 3037\n",
      "Sample string (Document 15) ['cinderella', 'the', 'wife', 'of', 'a', 'rich', 'man', 'fell', 'sick', ',', 'and', 'as', 'she', 'felt', 'that', 'her', 'end', 'was', 'drawing', 'near', ',', 'she', 'called', 'her', 'only', 'daughter', 'to', 'her', 'bedside', 'and', 'said', ',', 'dear', 'child', ',', 'be', 'good', 'and', 'pious', ',', 'and', 'then', 'the', 'good', 'god', 'will', 'always', 'protect', 'you', ',']\n",
      "\n",
      "Processing file stories\\017.txt\n",
      "Data size (Characters) (Document 16) 1320\n",
      "Sample string (Document 16) ['there', 'was', 'once', 'a', 'king', \"'s\", 'son', 'who', 'was', 'seized', 'with', 'a', 'desire', 'to', 'travel', 'about', 'the', 'world', ',', 'and', 'took', 'no', 'one', 'with', 'him', 'but', 'a', 'faithful', 'servant', '.', 'one', 'day', 'he', 'came', 'to', 'a', 'great', 'forest', ',', 'and', 'when', 'darkness', 'overtook', 'him', 'he', 'could', 'find', 'no', 'shelter', ',']\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing file stories\\018.txt\n",
      "Data size (Characters) (Document 17) 1370\n",
      "Sample string (Document 17) ['there', 'was', 'once', 'a', 'widow', 'who', 'had', 'two', 'daughters', '-', 'one', 'of', 'whom', 'was', 'pretty', 'and', 'industrious', ',', 'whilst', 'the', 'other', 'was', 'ugly', 'and', 'idle', '.', 'but', 'she', 'was', 'much', 'fonder', 'of', 'the', 'ugly', 'and', 'idle', 'one', ',', 'because', 'she', 'was', 'her', 'own', 'daughter', '.', 'and', 'the', 'other', ',', 'who']\n",
      "\n",
      "Processing file stories\\019.txt\n",
      "Data size (Characters) (Document 18) 1071\n",
      "Sample string (Document 18) ['there', 'was', 'once', 'a', 'man', 'who', 'had', 'seven', 'sons', ',', 'and', 'still', 'he', 'had', 'no', 'daughter', ',', 'however', 'much', 'he', 'wished', 'for', 'one', '.', 'at', 'length', 'his', 'wife', 'again', 'gave', 'him', 'hope', 'of', 'a', 'child', ',', 'and', 'when', 'it', 'came', 'into', 'the', 'world', 'it', 'was', 'a', 'girl', '.', 'the', 'joy']\n",
      "\n",
      "Processing file stories\\020.txt\n",
      "Data size (Characters) (Document 19) 1662\n",
      "Sample string (Document 19) ['little', 'red-cap', 'once', 'upon', 'a', 'time', 'there', 'was', 'a', 'dear', 'little', 'girl', 'who', 'was', 'loved', 'by', 'every', 'one', 'who', 'looked', 'at', 'her', ',', 'but', 'most', 'of', 'all', 'by', 'her', 'grandmother', ',', 'and', 'there', 'was', 'nothing', 'that', 'she', 'would', 'not', 'have', 'given', 'to', 'the', 'child', '.', 'once', 'she', 'gave', 'her', 'a']\n",
      "\n",
      "Processing file stories\\021.txt\n",
      "Data size (Characters) (Document 20) 835\n",
      "Sample string (Document 20) ['in', 'a', 'certain', 'country', 'there', 'was', 'once', 'great', 'lamentation', 'over', 'a', 'wild', 'boar', 'that', 'laid', 'waste', 'the', 'farmer', \"'s\", 'fields', ',', 'killed', 'the', 'cattle', ',', 'and', 'ripped', 'up', 'people', \"'s\", 'bodies', 'with', 'his', 'tusks', '.', 'the', 'king', 'promised', 'a', 'large', 'reward', 'to', 'anyone', 'who', 'would', 'free', 'the', 'land', 'from', 'this']\n",
      "\n",
      "Processing file stories\\022.txt\n",
      "Data size (Characters) (Document 21) 2969\n",
      "Sample string (Document 21) ['there', 'was', 'once', 'a', 'poor', 'woman', 'who', 'gave', 'birth', 'to', 'a', 'little', 'son', ',', 'and', 'as', 'he', 'came', 'into', 'the', 'world', 'with', 'a', 'caul', 'on', ',', 'it', 'was', 'predicted', 'that', 'in', 'his', 'fourteenth', 'year', 'he', 'would', 'have', 'the', 'king', \"'s\", 'daughter', 'for', 'his', 'wife', '.', 'it', 'happened', 'that', 'soon', 'afterwards']\n",
      "\n",
      "Processing file stories\\023.txt\n",
      "Data size (Characters) (Document 22) 2717\n",
      "Sample string (Document 22) ['a', 'certain', 'miller', 'had', 'little', 'by', 'little', 'fallen', 'into', 'poverty', ',', 'and', 'had', 'nothing', 'left', 'but', 'his', 'mill', 'and', 'a', 'large', 'apple-tree', 'behind', 'it', '.', 'once', 'when', 'he', 'had', 'gone', 'into', 'the', 'forest', 'to', 'fetch', 'wood', ',', 'an', 'old', 'man', 'stepped', 'up', 'to', 'him', 'whom', 'he', 'had', 'never', 'seen', 'before']\n",
      "\n",
      "Processing file stories\\024.txt\n",
      "Data size (Characters) (Document 23) 1202\n",
      "Sample string (Document 23) ['the', 'mother', 'of', 'hans', 'said', ',', 'whither', 'away', ',', 'hans', '.', 'hans', 'answered', ',', 'to', 'gretel', '.', 'behave', 'well', ',', 'hans', '.', 'oh', ',', 'i', \"'ll\", 'behave', 'well', '.', 'good-bye', ',', 'mother', '.', 'good-bye', ',', 'hans', '.', 'hans', 'comes', 'to', 'gretel', '.', 'good', 'day', ',', 'gretel', '.', 'good', 'day', ',']\n",
      "\n",
      "Processing file stories\\025.txt\n",
      "Data size (Characters) (Document 24) 1090\n",
      "Sample string (Document 24) ['an', 'aged', 'count', 'once', 'lived', 'in', 'switzerland', ',', 'who', 'had', 'an', 'only', 'son', ',', 'but', 'he', 'was', 'stupid', ',', 'and', 'could', 'learn', 'nothing', '.', 'then', 'said', 'the', 'father', ',', 'hark', 'you', ',', 'my', 'son', ',', 'try', 'as', 'i', 'will', 'i', 'can', 'get', 'nothing', 'into', 'your', 'head', '.', 'you', 'must', 'go']\n",
      "\n",
      "Processing file stories\\026.txt\n",
      "Data size (Characters) (Document 25) 1550\n",
      "Sample string (Document 25) ['there', 'was', 'once', 'a', 'man', 'who', 'had', 'a', 'daughter', 'who', 'was', 'called', 'clever', 'elsie', '.', 'and', 'when', 'she', 'had', 'grown', 'up', 'her', 'father', 'said', ',', 'we', 'will', 'get', 'her', 'married', '.', 'yes', ',', 'said', 'the', 'mother', ',', 'if', 'only', 'someone', 'would', 'come', 'who', 'would', 'have', 'her', '.', 'at', 'length', 'a']\n",
      "\n",
      "Processing file stories\\027.txt\n",
      "Data size (Characters) (Document 26) 4447\n",
      "Sample string (Document 26) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'tailor', 'who', 'had', 'three', 'sons', ',', 'and', 'only', 'one', 'goat', '.', 'but', 'as', 'the', 'goat', 'supported', 'all', 'of', 'them', 'with', 'her', 'milk', ',', 'she', 'was', 'obliged', 'to', 'have', 'good', 'food', ',', 'and', 'to', 'be', 'taken', 'every', 'day', 'to', 'pasture', '.', 'the', 'sons', 'did']\n",
      "\n",
      "Processing file stories\\028.txt\n",
      "Data size (Characters) (Document 27) 2638\n",
      "Sample string (Document 27) ['there', 'was', 'once', 'a', 'poor', 'peasant', 'who', 'sat', 'in', 'the', 'evening', 'by', 'the', 'hearth', 'and', 'poked', 'the', 'fire', ',', 'and', 'his', 'wife', 'sat', 'and', 'spun', '.', 'then', 'said', 'he', ',', 'how', 'sad', 'it', 'is', 'that', 'we', 'have', 'no', 'children', '.', 'with', 'us', 'all', 'is', 'so', 'quiet', ',', 'and', 'in', 'other']\n",
      "\n",
      "Processing file stories\\029.txt\n",
      "Data size (Characters) (Document 28) 575\n",
      "Sample string (Document 28) ['there', 'was', 'once', 'a', 'poor', 'servant-girl', 'who', 'was', 'industrious', 'and', 'cleanly', 'and', 'swept', 'the', 'house', 'every', 'day', ',', 'and', 'emptied', 'her', 'sweepings', 'on', 'the', 'great', 'heap', 'in', 'front', 'of', 'the', 'door', '.', 'one', 'morning', 'when', 'she', 'was', 'just', 'going', 'back', 'to', 'her', 'work', ',', 'she', 'found', 'a', 'letter', 'on', 'this']\n",
      "\n",
      "Processing file stories\\030.txt\n",
      "Data size (Characters) (Document 29) 1611\n",
      "Sample string (Document 29) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'miller', ',', 'who', 'had', 'a', 'beautiful', 'daughter', ',', 'and', 'as', 'she', 'was', 'grown', 'up', ',', 'he', 'wished', 'that', 'she', 'was', 'provided', 'for', ',', 'and', 'well', 'married', '.', 'he', 'thought', ',', 'if', 'any', 'good', 'suitor', 'comes', 'and', 'asks', 'for', 'her', ',', 'i', 'will', 'give']\n",
      "\n",
      "Processing file stories\\031.txt\n",
      "Data size (Characters) (Document 30) 722\n",
      "Sample string (Document 30) ['a', 'poor', 'man', 'had', 'so', 'many', 'children', 'that', 'he', 'had', 'already', 'asked', 'everyone', 'in', 'the', 'world', 'to', 'be', 'godfather', ',', 'and', 'when', 'still', 'another', 'child', 'was', 'born', ',', 'no', 'one', 'else', 'was', 'left', 'whom', 'he', 'could', 'invite', '.', 'he', 'knew', 'not', 'what', 'to', 'do', ',', 'and', ',', 'in', 'his', 'perplexity']\n",
      "\n",
      "Processing file stories\\032.txt\n",
      "Data size (Characters) (Document 31) 354\n",
      "Sample string (Document 31) ['there', 'was', 'once', 'a', 'little', 'girl', 'who', 'was', 'obstinate', 'and', 'inquisitive', ',', 'and', 'when', 'her', 'parents', 'told', 'her', 'to', 'do', 'anything', ',', 'she', 'did', 'not', 'obey', 'them', ',', 'so', 'how', 'could', 'she', 'fare', 'well', '.', 'one', 'day', 'she', 'said', 'to', 'her', 'parents', ',', 'i', 'have', 'heard', 'so', 'much', 'of', 'frau']\n",
      "\n",
      "Processing file stories\\033.txt\n",
      "Data size (Characters) (Document 32) 1404\n",
      "Sample string (Document 32) ['a', 'poor', 'man', 'had', 'twelve', 'children', 'and', 'was', 'forced', 'to', 'work', 'night', 'and', 'day', 'to', 'give', 'them', 'even', 'bread', '.', 'when', 'therefore', 'the', 'thirteenth', 'came', 'into', 'the', 'world', ',', 'he', 'knew', 'not', 'what', 'to', 'do', 'in', 'his', 'trouble', ',', 'but', 'ran', 'out', 'into', 'the', 'great', 'highway', ',', 'and', 'resolved', 'to']\n",
      "\n",
      "Processing file stories\\034.txt\n",
      "Data size (Characters) (Document 33) 1835\n",
      "Sample string (Document 33) ['a', 'certain', 'tailor', 'had', 'a', 'son', ',', 'who', 'happened', 'to', 'be', 'small', ',', 'and', 'no', 'bigger', 'than', 'a', 'thumb', ',', 'and', 'on', 'this', 'account', 'he', 'was', 'always', 'called', 'thumbling', '.', 'he', 'had', ',', 'however', ',', 'some', 'courage', 'in', 'him', ',', 'and', 'said', 'to', 'his', 'father', ',', 'father', ',', 'i', 'must']\n",
      "\n",
      "Processing file stories\\035.txt\n",
      "Data size (Characters) (Document 34) 1614\n",
      "Sample string (Document 34) ['there', 'was', 'once', 'a', 'wizard', 'who', 'used', 'to', 'take', 'the', 'form', 'of', 'a', 'poor', 'man', ',', 'and', 'went', 'to', 'houses', 'and', 'begged', ',', 'and', 'caught', 'pretty', 'girls', '.', 'no', 'one', 'knew', 'whither', 'he', 'carried', 'them', ',', 'for', 'they', 'were', 'never', 'seen', 'again', '.', 'one', 'day', 'he', 'appeared', 'before', 'the', 'door']\n",
      "\n",
      "Processing file stories\\036.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 35) 3766\n",
      "Sample string (Document 35) ['it', 'is', 'now', 'long', 'ago', ',', 'quite', 'two', 'thousand', 'years', ',', 'since', 'there', 'was', 'a', 'rich', 'man', 'who', 'had', 'a', 'beautiful', 'and', 'pious', 'wife', ',', 'and', 'they', 'loved', 'each', 'other', 'dearly', '.', 'they', 'had', ',', 'however', ',', 'no', 'children', ',', 'though', 'they', 'wished', 'for', 'them', 'very', 'much', ',', 'and', 'the']\n",
      "\n",
      "Processing file stories\\037.txt\n",
      "Data size (Characters) (Document 36) 984\n",
      "Sample string (Document 36) ['a', 'farmer', 'once', 'had', 'a', 'faithful', 'dog', 'called', 'sultan', ',', 'who', 'had', 'grown', 'old', ',', 'and', 'lost', 'all', 'his', 'teeth', ',', 'so', 'that', 'he', 'could', 'no', 'longer', 'hold', 'on', 'to', 'anything', '.', 'one', 'day', 'the', 'farmer', 'was', 'standing', 'with', 'his', 'wife', 'before', 'the', 'house-door', ',', 'and', 'said', ',', 'to-morrow', 'i']\n",
      "\n",
      "Processing file stories\\038.txt\n",
      "Data size (Characters) (Document 37) 2250\n",
      "Sample string (Document 37) ['once', 'upon', 'a', 'time', ',', 'a', 'certain', 'king', 'was', 'hunting', 'in', 'a', 'great', 'forest', ',', 'and', 'he', 'chased', 'a', 'wild', 'beast', 'so', 'eagerly', 'that', 'none', 'of', 'his', 'attendants', 'could', 'follow', 'him', '.', 'when', 'evening', 'drew', 'near', 'he', 'stopped', 'and', 'looked', 'around', 'him', ',', 'and', 'then', 'he', 'saw', 'that', 'he', 'had']\n",
      "\n",
      "Processing file stories\\039.txt\n",
      "Data size (Characters) (Document 38) 1505\n",
      "Sample string (Document 38) ['briar-rose', 'a', 'long', 'time', 'ago', 'there', 'were', 'a', 'king', 'and', 'queen', 'who', 'said', 'every', 'day', ',', 'ah', ',', 'if', 'only', 'we', 'had', 'a', 'child', ',', 'but', 'they', 'never', 'had', 'one', '.', 'but', 'it', 'happened', 'that', 'once', 'when', 'the', 'queen', 'was', 'bathing', ',', 'a', 'frog', 'crept', 'out', 'of', 'the', 'water', 'on']\n",
      "\n",
      "Processing file stories\\040.txt\n",
      "Data size (Characters) (Document 39) 1115\n",
      "Sample string (Document 39) ['there', 'was', 'once', 'a', 'forester', 'who', 'went', 'into', 'the', 'forest', 'to', 'hunt', ',', 'and', 'as', 'he', 'entered', 'it', 'he', 'heard', 'a', 'sound', 'of', 'screaming', 'as', 'if', 'a', 'little', 'child', 'were', 'there', '.', 'he', 'followed', 'the', 'sound', ',', 'and', 'at', 'last', 'came', 'to', 'a', 'high', 'tree', ',', 'and', 'at', 'the', 'top']\n",
      "\n",
      "Processing file stories\\041.txt\n",
      "Data size (Characters) (Document 40) 1886\n",
      "Sample string (Document 40) ['a', 'king', 'had', 'a', 'daughter', 'who', 'was', 'beautiful', 'beyond', 'all', 'measure', ',', 'but', 'so', 'proud', 'and', 'haughty', 'withal', 'that', 'no', 'suitor', 'was', 'good', 'enough', 'for', 'her', '.', 'she', 'sent', 'away', 'one', 'after', 'the', 'other', ',', 'and', 'ridiculed', 'them', 'as', 'well', '.', 'once', 'the', 'king', 'made', 'a', 'great', 'feast', 'and', 'invited']\n",
      "\n",
      "Processing file stories\\042.txt\n",
      "Data size (Characters) (Document 41) 3616\n",
      "Sample string (Document 41) ['snow', 'white', 'and', 'the', 'seven', 'dwarfs', 'once', 'upon', 'a', 'time', 'in', 'the', 'middle', 'of', 'winter', ',', 'when', 'the', 'flakes', 'of', 'snow', 'were', 'falling', 'like', 'feathers', 'from', 'the', 'sky', ',', 'a', 'queen', 'sat', 'at', 'a', 'window', 'sewing', ',', 'and', 'the', 'frame', 'of', 'the', 'window', 'was', 'made', 'of', 'black', 'ebony', '.', 'and']\n",
      "\n",
      "Processing file stories\\043.txt\n",
      "Data size (Characters) (Document 42) 2662\n",
      "Sample string (Document 42) ['there', 'were', 'once', 'three', 'brothers', 'who', 'had', 'fallen', 'deeper', 'and', 'deeper', 'into', 'poverty', ',', 'and', 'at', 'last', 'their', 'need', 'was', 'so', 'great', 'that', 'they', 'had', 'to', 'endure', 'hunger', ',', 'and', 'had', 'nothing', 'to', 'eat', 'or', 'drink', '.', 'then', 'said', 'they', ',', 'it', 'can', 'not', 'go', 'on', 'like', 'this', ',', 'we']\n",
      "\n",
      "Processing file stories\\044.txt\n",
      "Data size (Characters) (Document 43) 1266\n",
      "Sample string (Document 43) ['rumpelstiltskin', 'once', 'there', 'was', 'a', 'miller', 'who', 'was', 'poor', ',', 'but', 'who', 'had', 'a', 'beautiful', 'daughter', '.', 'now', 'it', 'happened', 'that', 'he', 'had', 'to', 'go', 'and', 'speak', 'to', 'the', 'king', ',', 'and', 'in', 'order', 'to', 'make', 'himself', 'appear', 'important', 'he', 'said', 'to', 'him', ',', 'i', 'have', 'a', 'daughter', 'who', 'can']\n",
      "\n",
      "Processing file stories\\045.txt\n",
      "Data size (Characters) (Document 44) 1709\n",
      "Sample string (Document 44) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'woman', 'who', 'was', 'a', 'real', 'witch', 'and', 'had', 'two', 'daughters', ',', 'one', 'ugly', 'and', 'wicked', ',', 'and', 'this', 'one', 'she', 'loved', 'because', 'she', 'was', 'her', 'own', 'daughter', ',', 'and', 'one', 'beautiful', 'and', 'good', ',', 'and', 'this', 'one', 'she', 'hated', ',', 'because', 'she', 'was']\n",
      "\n",
      "Processing file stories\\046.txt\n",
      "Data size (Characters) (Document 45) 3493\n",
      "Sample string (Document 45) ['in', 'olden', 'times', 'there', 'was', 'a', 'king', ',', 'who', 'had', 'behind', 'his', 'palace', 'a', 'beautiful', 'pleasure-garden', 'in', 'which', 'there', 'was', 'a', 'tree', 'that', 'bore', 'golden', 'apples', '.', 'when', 'the', 'apples', 'were', 'getting', 'ripe', 'they', 'were', 'counted', ',', 'but', 'on', 'the', 'very', 'next', 'morning', 'one', 'was', 'missing', '.', 'this', 'was', 'told']\n",
      "\n",
      "Processing file stories\\047.txt\n",
      "Data size (Characters) (Document 46) 9901\n",
      "Sample string (Document 46) ['there', 'were', 'once', 'upon', 'a', 'time', 'two', 'brothers', ',', 'one', 'rich', 'and', 'the', 'other', 'poor', '.', 'the', 'rich', 'one', 'was', 'a', 'goldsmith', 'and', 'evil-hearted', '.', 'the', 'poor', 'one', 'supported', 'himself', 'by', 'making', 'brooms', ',', 'and', 'was', 'good', 'and', 'honorable', '.', 'he', 'had', 'two', 'children', ',', 'who', 'were', 'twin', 'brothers', 'and']\n",
      "\n",
      "Processing file stories\\048.txt\n",
      "Data size (Characters) (Document 47) 934\n",
      "Sample string (Document 47) ['two', 'kings', \"'\", 'sons', 'once', 'went', 'out', 'in', 'search', 'of', 'adventures', ',', 'and', 'fell', 'into', 'a', 'wild', ',', 'disorderly', 'way', 'of', 'living', ',', 'so', 'that', 'they', 'never', 'came', 'home', 'again', '.', 'the', 'youngest', ',', 'who', 'was', 'called', 'simpleton', ',', 'set', 'out', 'to', 'seek', 'his', 'brothers', ',', 'but', 'when', 'at', 'length']\n",
      "\n",
      "Processing file stories\\049.txt\n",
      "Data size (Characters) (Document 48) 1180\n",
      "Sample string (Document 48) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'king', 'who', 'had', 'three', 'sons', ',', 'of', 'whom', 'two', 'were', 'clever', 'and', 'wise', ',', 'but', 'the', 'third', 'did', 'not', 'speak', 'much', ',', 'and', 'was', 'simple', ',', 'and', 'was', 'called', 'the', 'simpleton', '.', 'when', 'the', 'king', 'had', 'become', 'old', 'and', 'weak', ',', 'and', 'was']\n",
      "\n",
      "Processing file stories\\050.txt\n",
      "Data size (Characters) (Document 49) 1792\n",
      "Sample string (Document 49) ['there', 'was', 'a', 'man', 'who', 'had', 'three', 'sons', ',', 'the', 'youngest', 'of', 'whom', 'was', 'called', 'dummling', ',', 'and', 'was', 'despised', ',', 'mocked', ',', 'and', 'sneered', 'at', 'on', 'every', 'occasion', '.', 'it', 'happened', 'that', 'the', 'eldest', 'wanted', 'to', 'go', 'into', 'the', 'forest', 'to', 'hew', 'wood', ',', 'and', 'before', 'he', 'went', 'his']\n",
      "\n",
      "Processing file stories\\051.txt\n",
      "Data size (Characters) (Document 50) 2503\n",
      "Sample string (Document 50) ['allerleirauh', 'there', 'was', 'once', 'upon', 'a', 'time', 'a', 'king', 'who', 'had', 'a', 'wife', 'with', 'golden', 'hair', ',', 'and', 'she', 'was', 'so', 'beautiful', 'that', 'her', 'equal', 'was', 'not', 'to', 'be', 'found', 'on', 'earth', '.', 'it', 'came', 'to', 'pass', 'that', 'she', 'lay', 'ill', ',', 'and', 'as', 'she', 'felt', 'that', 'she', 'must', 'soon']\n",
      "\n",
      "Processing file stories\\052.txt\n",
      "Data size (Characters) (Document 51) 579\n",
      "Sample string (Document 51) ['there', 'was', 'once', 'a', 'woman', 'and', 'her', 'daughter', 'who', 'lived', 'in', 'a', 'pretty', 'garden', 'with', 'cabbages', '.', 'and', 'a', 'little', 'hare', 'came', 'into', 'it', ',', 'and', 'during', 'the', 'winter', 'time', 'ate', 'all', 'the', 'cabbages', '.', 'then', 'says', 'the', 'mother', 'to', 'the', 'daughter', ',', 'go', 'into', 'the', 'garden', ',', 'and', 'chase']\n",
      "\n",
      "Processing file stories\\053.txt\n",
      "Data size (Characters) (Document 52) 1249\n",
      "Sample string (Document 52) ['there', 'was', 'once', 'a', 'king', \"'s\", 'son', 'who', 'had', 'a', 'bride', 'whom', 'he', 'loved', 'very', 'much', '.', 'and', 'when', 'he', 'was', 'sitting', 'beside', 'her', 'and', 'very', 'happy', ',', 'news', 'came', 'that', 'his', 'father', 'lay', 'sick', 'unto', 'death', ',', 'and', 'desired', 'to', 'see', 'him', 'once', 'again', 'before', 'his', 'end', '.', 'then']\n",
      "\n",
      "Processing file stories\\054.txt\n",
      "Data size (Characters) (Document 53) 890\n",
      "Sample string (Document 53) ['hans', 'wished', 'to', 'put', 'his', 'son', 'to', 'learn', 'a', 'trade', ',', 'so', 'he', 'went', 'into', 'the', 'church', 'and', 'prayed', 'to', 'our', 'lord', 'god', 'to', 'know', 'which', 'would', 'be', 'the', 'most', 'suitable', 'for', 'him', '.', 'then', 'the', 'clerk', 'got', 'behind', 'the', 'altar', ',', 'and', 'said', ',', 'thieving', ',', 'thieving', '.', 'on']\n",
      "\n",
      "Processing file stories\\055.txt\n",
      "Data size (Characters) (Document 54) 1117\n",
      "Sample string (Document 54) ['a', 'father', 'once', 'called', 'his', 'three', 'sons', 'before', 'him', ',', 'and', 'he', 'gave', 'to', 'the', 'first', 'a', 'cock', ',', 'to', 'the', 'second', 'a', 'scythe', ',', 'and', 'to', 'the', 'third', 'a', 'cat', '.', 'i', 'am', 'already', 'aged', ',', 'said', 'he', ',', 'my', 'death', 'is', 'nigh', ',', 'and', 'i', 'have', 'wished', 'to']\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file stories\\056.txt\n",
      "Data size (Characters) (Document 55) 2386\n",
      "Sample string (Document 55) ['there', 'was', 'once', 'a', 'man', 'who', 'understood', 'all', 'kinds', 'of', 'arts', '.', 'he', 'served', 'in', 'war', ',', 'and', 'behaved', 'well', 'and', 'bravely', ',', 'but', 'when', 'the', 'war', 'was', 'over', 'he', 'received', 'his', 'dismissal', ',', 'and', 'three', 'farthings', 'for', 'his', 'expenses', 'on', 'the', 'way', '.', 'wait', ',', 'said', 'he', ',', 'i']\n",
      "\n",
      "Processing file stories\\057.txt\n",
      "Data size (Characters) (Document 56) 441\n",
      "Sample string (Document 56) ['the', 'she-wolf', 'brought', 'into', 'the', 'world', 'a', 'young', 'one', ',', 'and', 'invited', 'the', 'fox', 'to', 'be', 'godfather', '.', 'after', 'all', ',', 'he', 'is', 'a', 'near', 'relative', 'of', 'ours', ',', 'said', 'she', ',', 'he', 'has', 'a', 'good', 'understanding', ',', 'and', 'much', 'talent', ',', 'he', 'can', 'instruct', 'my', 'little', 'son', ',', 'and']\n",
      "\n",
      "Processing file stories\\058.txt\n",
      "Data size (Characters) (Document 57) 2052\n",
      "Sample string (Document 57) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'queen', 'to', 'whom', 'god', 'had', 'given', 'no', 'children', '.', 'every', 'morning', 'she', 'went', 'into', 'the', 'garden', 'and', 'prayed', 'to', 'god', 'in', 'heaven', 'to', 'bestow', 'on', 'her', 'a', 'son', 'or', 'a', 'daughter', '.', 'then', 'an', 'angel', 'from', 'heaven', 'came', 'to', 'her', 'and', 'said', ',']\n",
      "\n",
      "Processing file stories\\059.txt\n",
      "Data size (Characters) (Document 58) 280\n",
      "Sample string (Document 58) ['there', 'was', 'once', 'a', 'very', 'old', 'man', ',', 'whose', 'eyes', 'had', 'become', 'dim', ',', 'his', 'ears', 'dull', 'of', 'hearing', ',', 'his', 'knees', 'trembled', ',', 'and', 'when', 'he', 'sat', 'at', 'table', 'he', 'could', 'hardly', 'hold', 'the', 'spoon', ',', 'and', 'spilt', 'the', 'broth', 'upon', 'the', 'table-cloth', 'or', 'let', 'it', 'run', 'out', 'of']\n",
      "\n",
      "Processing file stories\\060.txt\n",
      "Data size (Characters) (Document 59) 338\n",
      "Sample string (Document 59) ['a', 'little', 'brother', 'and', 'sister', 'were', 'once', 'playing', 'by', 'a', 'well', ',', 'and', 'while', 'they', 'were', 'thus', 'playing', ',', 'they', 'both', 'fell', 'in', '.', 'a', 'water-nixie', 'lived', 'down', 'below', ',', 'who', 'said', ',', 'now', 'i', 'have', 'got', 'you', ',', 'now', 'you', 'shall', 'work', 'hard', 'for', 'me', ',', 'and', 'carried', 'them']\n",
      "\n",
      "Processing file stories\\061.txt\n",
      "Data size (Characters) (Document 60) 4882\n",
      "Sample string (Document 60) ['there', 'was', 'one', 'upon', 'a', 'time', 'a', 'great', 'war', ',', 'and', 'when', 'it', 'came', 'to', 'an', 'end', ',', 'many', 'soldiers', 'were', 'discharged', '.', 'then', 'brother', 'lustig', 'also', 'received', 'his', 'dismissal', ',', 'and', 'with', 'it', 'nothing', 'but', 'a', 'small', 'loaf', 'of', 'ammunition-bread', ',', 'and', 'four', 'kreuzers', 'in', 'money', ',', 'with', 'which']\n",
      "\n",
      "Processing file stories\\062.txt\n",
      "Data size (Characters) (Document 61) 2355\n",
      "Sample string (Document 61) ['hans', 'had', 'served', 'his', 'master', 'for', 'seven', 'years', ',', 'so', 'he', 'said', 'to', 'him', ',', 'master', ',', 'my', 'time', 'is', 'up', ',', 'now', 'i', 'should', 'be', 'glad', 'to', 'go', 'back', 'home', 'to', 'my', 'mother', ',', 'give', 'me', 'my', 'wages', '.', 'the', 'master', 'answered', ',', 'you', 'have', 'served', 'me', 'faithfully', 'and']\n",
      "\n",
      "Processing file stories\\063.txt\n",
      "Data size (Characters) (Document 62) 512\n",
      "Sample string (Document 62) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'young', 'peasant', 'named', 'hans', ',', 'whose', 'uncle', 'wanted', 'to', 'find', 'him', 'a', 'rich', 'wife', '.', 'he', 'therefore', 'seated', 'hans', 'behind', 'the', 'stove', ',', 'and', 'had', 'it', 'made', 'very', 'hot', '.', 'then', 'he', 'fetched', 'a', 'pot', 'of', 'milk', 'and', 'plenty', 'of', 'white', 'bread', ',']\n",
      "\n",
      "Processing file stories\\064.txt\n",
      "Data size (Characters) (Document 63) 2230\n",
      "Sample string (Document 63) ['there', 'was', 'once', 'a', 'poor', 'man', 'and', 'a', 'poor', 'woman', 'who', 'had', 'nothing', 'but', 'a', 'little', 'cottage', ',', 'and', 'who', 'earned', 'their', 'bread', 'by', 'fishing', ',', 'and', 'always', 'lived', 'from', 'hand', 'to', 'mouth', '.', 'but', 'it', 'came', 'to', 'pass', 'one', 'day', 'when', 'the', 'man', 'was', 'sitting', 'by', 'the', 'water-side', ',']\n",
      "\n",
      "Processing file stories\\065.txt\n",
      "Data size (Characters) (Document 64) 2672\n",
      "Sample string (Document 64) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'man', 'who', 'was', 'about', 'to', 'set', 'out', 'on', 'a', 'long', 'journey', ',', 'and', 'on', 'parting', 'he', 'asked', 'his', 'three', 'daughters', 'what', 'he', 'should', 'bring', 'back', 'with', 'him', 'for', 'them', '.', 'whereupon', 'the', 'eldest', 'wished', 'for', 'pearls', ',', 'the', 'second', 'wished', 'for', 'diamonds', ',']\n",
      "\n",
      "Processing file stories\\066.txt\n",
      "Data size (Characters) (Document 65) 2543\n",
      "Sample string (Document 65) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'queen', 'whose', 'husband', 'had', 'been', 'dead', 'for', 'many', 'years', ',', 'and', 'she', 'had', 'a', 'beautiful', 'daughter', '.', 'when', 'the', 'princess', 'grew', 'up', 'she', 'was', 'betrothed', 'to', 'a', 'prince', 'who', 'lived', 'at', 'a', 'great', 'distance', '.', 'when', 'the', 'time', 'came', 'for', 'her', 'to']\n",
      "\n",
      "Processing file stories\\067.txt\n",
      "Data size (Characters) (Document 66) 3592\n",
      "Sample string (Document 66) ['once', 'upon', 'a', 'time', 'a', 'countryman', 'had', 'a', 'son', 'who', 'was', 'as', 'big', 'as', 'a', 'thumb', ',', 'and', 'did', 'not', 'become', 'any', 'bigger', ',', 'and', 'during', 'several', 'years', 'did', 'not', 'grow', 'one', 'hair', \"'s\", 'breadth', '.', 'once', 'when', 'the', 'father', 'was', 'going', 'out', 'to', 'plough', ',', 'the', 'little', 'one', 'said']\n",
      "\n",
      "Processing file stories\\068.txt\n",
      "Data size (Characters) (Document 67) 2088\n",
      "Sample string (Document 67) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'rich', 'king', 'who', 'had', 'three', 'daughters', ',', 'who', 'daily', 'went', 'to', 'walk', 'in', 'the', 'palace', 'garden', ',', 'and', 'the', 'king', 'was', 'a', 'great', 'lover', 'of', 'all', 'kinds', 'of', 'fine', 'trees', ',', 'but', 'there', 'was', 'one', 'for', 'which', 'he', 'had', 'such', 'an', 'affection', ',']\n",
      "\n",
      "Processing file stories\\069.txt\n",
      "Data size (Characters) (Document 68) 2814\n",
      "Sample string (Document 68) ['there', 'was', 'a', 'certain', 'merchant', 'who', 'had', 'two', 'children', ',', 'a', 'boy', 'and', 'a', 'girl', ',', 'they', 'were', 'both', 'young', ',', 'and', 'could', 'not', 'walk', '.', 'and', 'two', 'richly-laden', 'ships', 'of', 'his', 'sailed', 'forth', 'to', 'sea', 'with', 'all', 'his', 'property', 'on', 'board', ',', 'and', 'just', 'as', 'he', 'was', 'expecting', 'to']\n",
      "\n",
      "Processing file stories\\070.txt\n",
      "Data size (Characters) (Document 69) 2587\n",
      "Sample string (Document 69) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'queen', 'who', 'had', 'a', 'little', 'daughter', 'who', 'was', 'still', 'so', 'young', 'that', 'she', 'had', 'to', 'be', 'carried', '.', 'one', 'day', 'the', 'child', 'was', 'naughty', ',', 'and', 'the', 'mother', 'might', 'say', 'what', 'she', 'liked', ',', 'but', 'the', 'child', 'would', 'not', 'be', 'quiet', '.', 'then']\n",
      "\n",
      "Processing file stories\\071.txt\n",
      "Data size (Characters) (Document 70) 1619\n",
      "Sample string (Document 70) ['there', 'was', 'once', 'a', 'poor', 'peasant', 'who', 'had', 'no', 'land', ',', 'but', 'only', 'a', 'small', 'house', ',', 'and', 'one', 'daughter', '.', 'then', 'said', 'the', 'daughter', ',', 'we', 'ought', 'to', 'ask', 'our', 'lord', 'the', 'king', 'for', 'a', 'bit', 'of', 'newly-cleared', 'land', '.', 'when', 'the', 'king', 'heard', 'of', 'their', 'poverty', ',', 'he']\n",
      "\n",
      "Processing file stories\\072.txt\n",
      "Data size (Characters) (Document 71) 1679\n",
      "Sample string (Document 71) ['about', 'a', 'thousand', 'or', 'more', 'years', 'ago', ',', 'there', 'were', 'in', 'this', 'country', 'nothing', 'but', 'small', 'kings', ',', 'and', 'one', 'of', 'them', 'who', 'lived', 'on', 'the', 'keuterberg', 'was', 'very', 'fond', 'of', 'hunting', '.', 'once', 'on', 'a', 'time', 'when', 'he', 'was', 'riding', 'forth', 'from', 'his', 'castle', 'with', 'his', 'huntsmen', ',', 'three']\n",
      "\n",
      "Processing file stories\\073.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 72) 2708\n",
      "Sample string (Document 72) ['there', 'was', 'once', 'a', 'king', 'who', 'had', 'an', 'illness', ',', 'and', 'no', 'one', 'believed', 'that', 'he', 'would', 'come', 'out', 'of', 'it', 'with', 'his', 'life', '.', 'he', 'had', 'three', 'sons', 'who', 'were', 'much', 'distressed', 'about', 'it', ',', 'and', 'went', 'down', 'into', 'the', 'palace-garden', 'and', 'wept', '.', 'there', 'they', 'met', 'an', 'old']\n",
      "\n",
      "Processing file stories\\074.txt\n",
      "Data size (Characters) (Document 73) 2203\n",
      "Sample string (Document 73) ['there', 'was', 'once', 'a', 'poor', 'woodcutter', 'who', 'toiled', 'from', 'early', 'morning', 'till', 'late', 'at', 'night', '.', 'when', 'at', 'last', 'he', 'had', 'laid', 'by', 'some', 'money', 'he', 'said', 'to', 'his', 'boy', ',', '``', 'you', 'are', 'my', 'only', 'child', ',', 'i', 'will', 'spend', 'the', 'money', 'which', 'i', 'have', 'earned', 'with', 'the', 'sweat']\n",
      "\n",
      "Processing file stories\\075.txt\n",
      "Data size (Characters) (Document 74) 1522\n",
      "Sample string (Document 74) ['a', 'discharged', 'soldier', 'had', 'nothing', 'to', 'live', 'on', ',', 'and', 'did', 'not', 'know', 'how', 'to', 'make', 'his', 'way', '.', 'so', 'he', 'went', 'out', 'into', 'the', 'forest', 'and', 'when', 'he', 'had', 'walked', 'for', 'a', 'short', 'time', ',', 'he', 'met', 'a', 'little', 'man', 'who', 'turned', 'out', 'to', 'be', 'the', 'devil', '.', 'the']\n",
      "\n",
      "Processing file stories\\076.txt\n",
      "Data size (Characters) (Document 75) 2345\n",
      "Sample string (Document 75) ['there', 'was', 'once', 'a', 'young', 'fellow', 'who', 'enlisted', 'as', 'a', 'soldier', ',', 'conducted', 'himself', 'bravely', ',', 'and', 'was', 'always', 'the', 'foremost', 'when', 'it', 'rained', 'bullets', '.', 'so', 'long', 'as', 'the', 'war', 'lasted', ',', 'all', 'went', 'well', ',', 'but', 'when', 'peace', 'was', 'made', ',', 'he', 'received', 'his', 'dismissal', ',', 'and', 'the']\n",
      "\n",
      "Processing file stories\\077.txt\n",
      "Data size (Characters) (Document 76) 1108\n",
      "Sample string (Document 76) ['once', 'in', 'summer-time', 'the', 'bear', 'and', 'the', 'wolf', 'were', 'walking', 'in', 'the', 'forest', ',', 'and', 'the', 'bear', 'heard', 'a', 'bird', 'singing', 'so', 'beautifully', 'that', 'he', 'said', ',', \"''\", 'brother', 'wolf', ',', 'what', 'bird', 'is', 'it', 'that', 'sings', 'so', 'well', '?', \"''\", '``', 'that', 'is', 'the', 'king', 'of', 'birds', ',', \"''\"]\n",
      "\n",
      "Processing file stories\\078.txt\n",
      "Data size (Characters) (Document 77) 284\n",
      "Sample string (Document 77) ['there', 'was', 'a', 'poor', 'but', 'good', 'little', 'girl', 'who', 'lived', 'alone', 'with', 'her', 'mother', ',', 'and', 'they', 'no', 'longer', 'had', 'anything', 'to', 'eat', '.', 'so', 'the', 'child', 'went', 'into', 'the', 'forest', ',', 'and', 'there', 'an', 'aged', 'woman', 'met', 'her', 'who', 'was', 'aware', 'of', 'her', 'sorrow', ',', 'and', 'presented', 'her', 'with']\n",
      "\n",
      "Processing file stories\\079.txt\n",
      "Data size (Characters) (Document 78) 1822\n",
      "Sample string (Document 78) ['one', 'day', 'a', 'peasant', 'took', 'his', 'good', 'hazel-stick', 'out', 'of', 'the', 'corner', 'and', 'said', 'to', 'his', 'wife', ',', 'trina', ',', 'i', 'am', 'going', 'across', 'country', ',', 'and', 'shall', 'not', 'return', 'for', 'three', 'days', '.', 'if', 'during', 'that', 'time', 'the', 'cattle-dealer', 'should', 'happen', 'to', 'call', 'and', 'want', 'to', 'buy', 'our', 'three']\n",
      "\n",
      "Processing file stories\\080.txt\n",
      "Data size (Characters) (Document 79) 588\n",
      "Sample string (Document 79) ['there', 'was', 'once', 'a', 'little', 'child', 'whose', 'mother', 'gave', 'her', 'every', 'afternoon', 'a', 'small', 'bowl', 'of', 'milk', 'and', 'bread', ',', 'and', 'the', 'child', 'seated', 'herself', 'in', 'the', 'yard', 'with', 'it', '.', 'but', 'when', 'she', 'began', 'to', 'eat', ',', 'a', 'paddock', 'came', 'creeping', 'out', 'of', 'a', 'crevice', 'in', 'the', 'wall', ',']\n",
      "\n",
      "Processing file stories\\081.txt\n",
      "Data size (Characters) (Document 80) 1670\n",
      "Sample string (Document 80) ['in', 'a', 'certain', 'mill', 'lived', 'an', 'old', 'miller', 'who', 'had', 'neither', 'wife', 'nor', 'child', ',', 'and', 'three', 'apprentices', 'served', 'under', 'him', '.', 'as', 'they', 'had', 'been', 'with', 'him', 'several', 'years', ',', 'he', 'one', 'day', 'said', 'to', 'them', ',', '``', 'i', 'am', 'old', ',', 'and', 'want', 'to', 'sit', 'behind', 'the', 'stove']\n",
      "\n",
      "Processing file stories\\082.txt\n",
      "Data size (Characters) (Document 81) 4887\n",
      "Sample string (Document 81) ['hill', 'and', 'vale', 'do', 'not', 'meet', ',', 'but', 'the', 'children', 'of', 'men', 'do', ',', 'good', 'and', 'bad', '.', 'in', 'this', 'way', 'a', 'shoemaker', 'and', 'a', 'tailor', 'once', 'met', 'on', 'their', 'travels', '.', 'the', 'tailor', 'was', 'a', 'handsome', 'little', 'fellow', 'who', 'was', 'always', 'merry', 'and', 'full', 'of', 'enjoyment', '.', 'he', 'saw']\n",
      "\n",
      "Processing file stories\\083.txt\n",
      "Data size (Characters) (Document 82) 2435\n",
      "Sample string (Document 82) ['hans', 'the', 'hedgehog', 'there', 'was', 'once', 'a', 'country', 'man', 'who', 'had', 'money', 'and', 'land', 'in', 'plenty', ',', 'but', 'however', 'rich', 'he', 'was', ',', 'his', 'happiness', 'was', 'still', 'lacking', 'in', 'one', 'respect', '-', 'he', 'had', 'no', 'children', '.', 'often', 'when', 'he', 'went', 'into', 'the', 'town', 'with', 'the', 'other', 'peasants', 'they', 'mocked']\n",
      "\n",
      "Processing file stories\\084.txt\n",
      "Data size (Characters) (Document 83) 298\n",
      "Sample string (Document 83) ['there', 'was', 'once', 'a', 'mother', 'who', 'had', 'a', 'little', 'boy', 'of', 'seven', 'years', 'old', ',', 'who', 'was', 'so', 'handsome', 'and', 'lovable', 'that', 'no', 'one', 'could', 'look', 'at', 'him', 'without', 'liking', 'him', ',', 'and', 'she', 'herself', 'worshipped', 'him', 'above', 'everything', 'in', 'the', 'world', '.', 'now', 'it', 'so', 'happened', 'that', 'he', 'suddenly']\n",
      "\n",
      "Processing file stories\\085.txt\n",
      "Data size (Characters) (Document 84) 2721\n",
      "Sample string (Document 84) ['there', 'was', 'once', 'a', 'young', 'fellow', 'who', 'had', 'learnt', 'the', 'trade', 'of', 'locksmith', ',', 'and', 'told', 'his', 'father', 'he', 'would', 'now', 'go', 'out', 'into', 'the', 'world', 'and', 'seek', 'his', 'fortune', '.', 'very', 'well', ',', 'said', 'the', 'father', ',', 'i', 'am', 'quite', 'content', 'with', 'that', ',', 'and', 'gave', 'him', 'some', 'money']\n",
      "\n",
      "Processing file stories\\086.txt\n",
      "Data size (Characters) (Document 85) 4011\n",
      "Sample string (Document 85) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'king', 'who', 'had', 'a', 'little', 'boy', 'in', 'whose', 'stars', 'it', 'had', 'been', 'foretold', 'that', 'he', 'should', 'be', 'killed', 'by', 'a', 'stag', 'when', 'he', 'was', 'sixteen', 'years', 'of', 'age', ',', 'and', 'when', 'he', 'had', 'reached', 'that', 'age', 'the', 'huntsmen', 'once', 'went', 'hunting', 'with', 'him']\n",
      "\n",
      "Processing file stories\\087.txt\n",
      "Data size (Characters) (Document 86) 1443\n",
      "Sample string (Document 86) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'princess', 'who', 'was', 'extremely', 'proud', '.', 'if', 'a', 'wooer', 'came', 'she', 'gave', 'him', 'some', 'riddle', 'to', 'guess', ',', 'and', 'if', 'he', 'could', 'not', 'guess', 'it', ',', 'he', 'was', 'sent', 'contemptuously', 'away', '.', 'she', 'let', 'it', 'be', 'made', 'known', 'also', 'that', 'whosoever', 'solved', 'her']\n",
      "\n",
      "Processing file stories\\088.txt\n",
      "Data size (Characters) (Document 87) 628\n",
      "Sample string (Document 87) ['a', 'tailor', \"'s\", 'apprentice', 'was', 'traveling', 'about', 'the', 'world', 'in', 'search', 'of', 'work', ',', 'and', 'at', 'one', 'time', 'he', 'could', 'find', 'none', ',', 'and', 'his', 'poverty', 'was', 'so', 'great', 'that', 'he', 'had', 'not', 'a', 'farthing', 'to', 'live', 'on', '.', 'presently', 'he', 'met', 'a', 'jew', 'on', 'the', 'road', ',', 'and', 'as']\n",
      "\n",
      "Processing file stories\\089.txt\n",
      "Data size (Characters) (Document 88) 2099\n",
      "Sample string (Document 88) ['there', 'was', 'once', 'on', 'a', 'time', 'a', 'soldier', 'who', 'for', 'many', 'years', 'had', 'served', 'the', 'king', 'faithfully', ',', 'but', 'when', 'the', 'war', 'came', 'to', 'an', 'end', 'could', 'serve', 'no', 'longer', 'because', 'of', 'the', 'many', 'wounds', 'which', 'he', 'had', 'received', '.', 'the', 'king', 'said', 'to', 'him', ',', '``', 'you', 'may', 'return']\n",
      "\n",
      "Processing file stories\\090.txt\n",
      "Data size (Characters) (Document 89) 160\n",
      "Sample string (Document 89) ['once', 'upon', 'a', 'time', 'there', 'was', 'a', 'child', 'who', 'was', 'willful', ',', 'and', 'would', 'not', 'do', 'what', 'her', 'mother', 'wished', '.', 'for', 'this', 'reason', 'god', 'had', 'no', 'pleasure', 'in', 'her', ',', 'and', 'let', 'her', 'become', 'ill', ',', 'and', 'no', 'doctor', 'could', 'do', 'her', 'any', 'good', ',', 'and', 'in', 'a', 'short']\n",
      "\n",
      "Processing file stories\\091.txt\n",
      "Data size (Characters) (Document 90) 2518\n",
      "Sample string (Document 90) ['there', 'was', 'once', 'a', 'king', \"'s\", 'son', ',', 'who', 'was', 'no', 'longer', 'content', 'to', 'stay', 'at', 'home', 'in', 'his', 'father', \"'s\", 'house', ',', 'and', 'as', 'he', 'had', 'no', 'fear', 'of', 'anything', ',', 'he', 'thought', ',', 'i', 'will', 'go', 'forth', 'into', 'the', 'wide', 'world', ',', 'there', 'the', 'time', 'will', 'not', 'seem']\n",
      "\n",
      "Processing file stories\\092.txt\n",
      "Data size (Characters) (Document 91) 3120\n",
      "Sample string (Document 91) ['there', 'was', 'once', 'a', 'young', 'huntsman', 'who', 'went', 'into', 'the', 'forest', 'to', 'lie', 'in', 'wait', '.', 'he', 'had', 'a', 'fresh', 'and', 'joyous', 'heart', ',', 'and', 'as', 'he', 'was', 'going', 'thither', ',', 'whistling', 'upon', 'a', 'leaf', ',', 'an', 'ugly', 'old', 'crone', 'came', 'up', ',', 'who', 'spoke', 'to', 'him', 'and', 'said', ',']\n",
      "\n",
      "Processing file stories\\093.txt\n",
      "Data size (Characters) (Document 92) 1077\n",
      "Sample string (Document 92) ['a', 'poor', 'servant-girl', 'was', 'once', 'traveling', 'with', 'the', 'family', 'with', 'which', 'she', 'was', 'in', 'service', ',', 'through', 'a', 'great', 'forest', ',', 'and', 'when', 'they', 'were', 'in', 'the', 'midst', 'of', 'it', ',', 'robbers', 'came', 'out', 'of', 'the', 'thicket', ',', 'and', 'murdered', 'all', 'they', 'found', '.', 'all', 'perished', 'together', 'except', 'the', 'girl']\n",
      "\n",
      "Processing file stories\\094.txt\n",
      "Data size (Characters) (Document 93) 778\n",
      "Sample string (Document 93) ['there', 'was', 'once', 'a', 'man', 'who', 'had', 'three', 'sons', ',', 'and', 'nothing', 'else', 'in', 'the', 'world', 'but', 'the', 'house', 'in', 'which', 'he', 'lived', '.', 'now', 'each', 'of', 'the', 'sons', 'wished', 'to', 'have', 'the', 'house', 'after', 'his', 'father', \"'s\", 'death', ',', 'but', 'the', 'father', 'loved', 'them', 'all', 'alike', ',', 'and', 'did']\n",
      "\n",
      "Processing file stories\\095.txt\n",
      "Data size (Characters) (Document 94) 1514\n",
      "Sample string (Document 94) ['there', 'was', 'a', 'great', 'war', ',', 'and', 'the', 'king', 'had', 'many', 'soldiers', ',', 'but', 'gave', 'them', 'small', 'pay', ',', 'so', 'small', 'that', 'they', 'could', 'not', 'live', 'upon', 'it', ',', 'so', 'three', 'of', 'them', 'agreed', 'among', 'themselves', 'to', 'desert', '.', 'one', 'of', 'them', 'said', 'to', 'the', 'others', ',', '``', 'if', 'we']\n",
      "\n",
      "Processing file stories\\096.txt\n",
      "Data size (Characters) (Document 95) 2369\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Sample string (Document 95) ['once', 'upon', 'a', 'time', 'lived', 'a', 'man', 'and', 'a', 'woman', 'who', 'so', 'long', 'as', 'they', 'were', 'rich', 'had', 'no', 'children', ',', 'but', 'when', 'they', 'were', 'poor', 'they', 'got', 'a', 'little', 'boy', '.', 'they', 'could', 'find', 'no', 'godfather', 'for', 'him', ',', 'so', 'the', 'man', 'said', 'he', 'would', 'just', 'go', 'to', 'another']\n",
      "\n",
      "Processing file stories\\097.txt\n",
      "Data size (Characters) (Document 96) 2613\n",
      "Sample string (Document 96) ['in', 'the', 'days', 'when', 'wishing', 'was', 'still', 'of', 'some', 'use', ',', 'a', 'king', \"'s\", 'son', 'was', 'bewitched', 'by', 'an', 'old', 'witch', ',', 'and', 'shut', 'up', 'in', 'an', 'iron', 'stove', 'in', 'a', 'forest', '.', 'there', 'he', 'passed', 'many', 'years', ',', 'and', 'no', 'one', 'could', 'rescue', 'him', '.', 'then', 'a', 'king', \"'s\"]\n",
      "\n",
      "Processing file stories\\098.txt\n",
      "Data size (Characters) (Document 97) 1980\n",
      "Sample string (Document 97) ['there', 'was', 'once', 'a', 'poor', 'man', 'who', 'had', 'four', 'sons', ',', 'and', 'when', 'they', 'were', 'grown', 'up', ',', 'he', 'said', 'to', 'them', ',', '``', 'my', 'dear', 'children', ',', 'you', 'must', 'now', 'go', 'out', 'into', 'the', 'world', ',', 'for', 'i', 'have', 'nothing', 'to', 'give', 'you', ',', 'so', 'set', 'out', ',', 'go']\n",
      "\n",
      "Processing file stories\\099.txt\n",
      "Data size (Characters) (Document 98) 3169\n",
      "Sample string (Document 98) ['there', 'was', 'once', 'a', 'woman', 'who', 'had', 'three', 'daughters', ',', 'the', 'eldest', 'of', 'whom', 'was', 'called', 'one-eye', ',', 'because', 'she', 'had', 'only', 'one', 'eye', 'in', 'the', 'middle', 'of', 'her', 'forehead', ',', 'and', 'the', 'second', ',', 'two-eyes', ',', 'because', 'she', 'had', 'two', 'eyes', 'like', 'other', 'folks', ',', 'and', 'the', 'youngest', ',']\n",
      "\n",
      "Processing file stories\\100.txt\n",
      "Data size (Characters) (Document 99) 492\n",
      "Sample string (Document 99) ['``', 'good-day', ',', 'father', 'hollenthe', '.', \"''\", '``', 'many', 'thanks', ',', 'pif-paf-poltrie', '.', \"''\", '``', 'may', 'i', 'be', 'allowed', 'to', 'have', 'your', 'daughter', '?', \"''\", '``', 'oh', ',', 'yes', ',', 'if', 'mother', 'malcho', 'milchcow', ',', 'brother', 'high-and-mighty', ',', 'sister', 'kasetraut', ',', 'and', 'fair', 'katrinelje', 'are', 'willing', ',', 'you', 'can', 'have']\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "  \n",
    "  with open(filename) as f:\n",
    "    data = tf.compat.as_str(f.read())\n",
    "    data = data.lower()\n",
    "    data = nltk.word_tokenize(data)\n",
    "    \n",
    "  return data\n",
    "\n",
    "documents = []\n",
    "global documents\n",
    "for i in range(num_files):    \n",
    "    print('\\nProcessing file %s'%os.path.join(dir_name,filenames[i]))\n",
    "    \n",
    "    words = read_data(os.path.join(dir_name,filenames[i]))\n",
    "    \n",
    "    documents.append(words)\n",
    "    print('Data size (Characters) (Document %d) %d' %(i,len(words)))\n",
    "    print('Sample string (Document %d) %s'%(i,words[:50]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the Dictionaries (Bigrams)\n",
    "Builds the following. To understand each of these elements, let us also assume the text \"I like to go to school\"\n",
    "\n",
    "* `dictionary`: maps a string word to an ID (e.g. {I:0, like:1, to:2, go:3, school:4})\n",
    "* `reverse_dictionary`: maps an ID to a string word (e.g. {0:I, 1:like, 2:to, 3:go, 4:school}\n",
    "* `count`: List of list of (word, frequency) elements (e.g. [(I,1),(like,1),(to,2),(go,1),(school,1)]\n",
    "* `data` : Contain the string of text we read, where string words are replaced with word IDs (e.g. [0, 1, 2, 3, 2, 4])\n",
    "\n",
    "It also introduces an additional special token `UNK` to denote rare words to are too rare to make use of."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "202036 Words found.\n",
      "Most common words (+UNK) [(',', 17882), ('the', 11960), ('and', 9325), ('.', 7633), ('to', 4520)]\n",
      "Least common words (+UNK) [('wet', 1), ('iii', 1), ('sheltered', 1), ('yelled', 1), ('oats', 1), ('armchair', 1), ('peaceably', 1), ('kitchen-sink', 1), ('fretted', 1), ('kernels', 1), ('youths', 1), ('hammers', 1), ('minutest', 1), ('tenderly', 1), ('craftily', 1)]\n",
      "Sample data [12, 0, 776, 25, 1173, 118, 826, 41, 1, 44]\n",
      "Sample data [577, 60, 7, 83, 119, 0, 7, 0, 22, 19]\n",
      "Vocabulary:  1322\n"
     ]
    }
   ],
   "source": [
    "def build_dataset(documents):\n",
    "    chars = []\n",
    "    # This is going to be a list of lists\n",
    "    # Where the outer list denote each document\n",
    "    # and the inner lists denote words in a given document\n",
    "    data_list = []\n",
    "  \n",
    "    for d in documents:\n",
    "        chars.extend(d)\n",
    "    print('%d Words found.'%len(chars))\n",
    "    count = []\n",
    "    # Get the word sorted by their frequency (Highest comes first)\n",
    "    count.extend(collections.Counter(chars).most_common())\n",
    "    \n",
    "    # Create an ID for each word by giving the current length of the dictionary\n",
    "    # And adding that item to the dictionary\n",
    "    # Start with 'UNK' that is assigned to too rare words\n",
    "    dictionary = dict({'UNK':0})\n",
    "    for char, c in count:\n",
    "        # Only add a bigram to dictionary if its frequency is more than 10\n",
    "        if c > 10:\n",
    "            dictionary[char] = len(dictionary)    \n",
    "    \n",
    "    unk_count = 0\n",
    "    # Traverse through all the text we have\n",
    "    # to replace each string word with the ID of the word\n",
    "    for d in documents:\n",
    "        data = list()\n",
    "        for char in d:\n",
    "            # If word is in the dictionary use the word ID,\n",
    "            # else use the ID of the special token \"UNK\"\n",
    "            if char in dictionary:\n",
    "                index = dictionary[char]        \n",
    "            else:\n",
    "                index = dictionary['UNK']\n",
    "                unk_count += 1\n",
    "            data.append(index)\n",
    "            \n",
    "        data_list.append(data)\n",
    "        \n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n",
    "    return data_list, count, dictionary, reverse_dictionary\n",
    "\n",
    "global data_list, count, dictionary, reverse_dictionary,vocabulary_size\n",
    "\n",
    "# Print some statistics about data\n",
    "data_list, count, dictionary, reverse_dictionary = build_dataset(documents)\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Least common words (+UNK)', count[-15:])\n",
    "print('Sample data', data_list[0][:10])\n",
    "print('Sample data', data_list[1][:10])\n",
    "print('Vocabulary: ',len(dictionary))\n",
    "vocabulary_size = len(dictionary)\n",
    "del documents  # To reduce memory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Batches of Data\n",
    "The following object generates a batch of data which will be used to train the RNN. More specifically the generator breaks a given sequence of words into `batch_size` segments. We also maintain a cursor for each segment. So whenever we create a batch of data, we sample one item from each segment and update the cursor of each segment. Note that the input words (as well as labels) are not one-hot encoded anymore as we need the word index to do embedding lookups."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Unrolled index 0\n",
      "\tInputs:\n",
      "\tthat (14), \twhich (56), \t, (1), \tshone (792), \tclose (822), \n",
      "\tOutput:\n",
      "\tthe (2), \thas (114), \twas (8), \tin (12), \tby (60), \n",
      "\n",
      "Unrolled index 1\n",
      "\tInputs:\n",
      "\tthe (2), \thas (114), \twas (8), \tin (12), \tby (60), \n",
      "\tOutput:\n",
      "\tsun (407), \tseen (283), \tastonished (869), \ther (17), \tthe (2), \n",
      "\n",
      "Unrolled index 2\n",
      "\tInputs:\n",
      "\tsun (407), \tseen (283), \tastonished (869), \ther (17), \tthe (2), \n",
      "\tOutput:\n",
      "\titself (720), \tso (33), \twhenever (1319), \tface (360), \tking (34), \n",
      "\n",
      "Unrolled index 3\n",
      "\tInputs:\n",
      "\titself (720), \tso (33), \twhenever (1319), \tface (360), \tking (34), \n",
      "\tOutput:\n",
      "\t, (1), \tmuch (165), \tit (9), \t. (4), \t's (53), \n",
      "\n",
      "Unrolled index 4\n",
      "\tInputs:\n",
      "\t, (1), \tmuch (165), \tit (9), \t. (4), \tclose (822), \n",
      "\tOutput:\n",
      "\twhich (56), \t, (1), \tshone (792), \tclose (822), \tby (60), "
     ]
    }
   ],
   "source": [
    "class DataGeneratorSeq(object):\n",
    "    \n",
    "    def __init__(self,text,batch_size,num_unroll):\n",
    "        # Text where a bigram is denoted by its ID\n",
    "        self._text = text\n",
    "        # Number of bigrams in the text\n",
    "        self._text_size = len(self._text)\n",
    "        # Number of datapoints in a batch of data\n",
    "        self._batch_size = batch_size\n",
    "        # Num unroll is the number of steps we unroll the RNN in a single training step\n",
    "        # This relates to the truncated backpropagation we discuss in Chapter 6 text\n",
    "        self._num_unroll = num_unroll\n",
    "        # We break the text in to several segments and the batch of data is sampled by\n",
    "        # sampling a single item from a single segment\n",
    "        self._segments = self._text_size//self._batch_size\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "    def next_batch(self):\n",
    "        '''\n",
    "        Generates a single batch of data\n",
    "        '''\n",
    "        # Train inputs (one-hot-encoded) and train outputs (one-hot-encoded)\n",
    "        batch_data = np.zeros((self._batch_size),dtype=np.float32)\n",
    "        batch_labels = np.zeros((self._batch_size),dtype=np.float32)\n",
    "        \n",
    "        # Fill in the batch datapoint by datapoint\n",
    "        for b in range(self._batch_size):\n",
    "            # If the cursor of a given segment exceeds the segment length\n",
    "            # we reset the cursor back to the beginning of that segment\n",
    "            if self._cursor[b]+1>=self._text_size:\n",
    "                self._cursor[b] = b * self._segments\n",
    "            \n",
    "            # Add the text at the cursor as the input\n",
    "            batch_data[b] = self._text[self._cursor[b]]\n",
    "            # Add the preceding word as the label to be predicted\n",
    "            batch_labels[b]= self._text[self._cursor[b]+1]                      \n",
    "            # Update the cursor\n",
    "            self._cursor[b] = (self._cursor[b]+1)%self._text_size\n",
    "                    \n",
    "        return batch_data,batch_labels\n",
    "        \n",
    "    def unroll_batches(self):\n",
    "        '''\n",
    "        This produces a list of num_unroll batches\n",
    "        as required by a single step of training of the RNN\n",
    "        '''\n",
    "        unroll_data,unroll_labels = [],[]\n",
    "        for ui in range(self._num_unroll):\n",
    "            data, labels = self.next_batch()            \n",
    "            unroll_data.append(data)\n",
    "            unroll_labels.append(labels)\n",
    "        \n",
    "        return unroll_data, unroll_labels\n",
    "    \n",
    "    def reset_indices(self):\n",
    "        '''\n",
    "        Used to reset all the cursors if needed\n",
    "        '''\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "# Running a tiny set to see if things are correct\n",
    "dg = DataGeneratorSeq(data_list[0][25:50],5,5)\n",
    "u_data, u_labels = dg.unroll_batches()\n",
    "\n",
    "# Iterate through each data batch in the unrolled set of batches\n",
    "for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):   \n",
    "    print('\\n\\nUnrolled index %d'%ui)\n",
    "    dat_ind = dat\n",
    "    lbl_ind = lbl\n",
    "    print('\\tInputs:')\n",
    "    for sing_dat in dat_ind:\n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_dat],sing_dat),end=\", \")\n",
    "    print('\\n\\tOutput:')\n",
    "    for sing_lbl in lbl_ind:        \n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_lbl],sing_lbl),end=\", \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the LSTM\n",
    "\n",
    "This is a standard LSTM. The LSTM has 5 main components.\n",
    "* Cell state\n",
    "* Hidden state\n",
    "* Input gate\n",
    "* Forget gate\n",
    "* Output gate\n",
    "\n",
    "Each gate has three sets of weights (1 set for the current input, 1 set for the previous hidden state and 1 bias)\n",
    "\n",
    "## Defining hyperparameters\n",
    "\n",
    "Here we define several hyperparameters and are very similar to the ones we defined in Chapter 6. However additionally we use dropout; a technique that helps to avoid overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Number of neurons in the hidden state variables\n",
    "num_nodes = [64, 48, 32]\n",
    "\n",
    "# Number of data points in a batch we process\n",
    "batch_size = 32\n",
    "\n",
    "# Number of time steps we unroll for during optimization\n",
    "num_unrollings = 50\n",
    "\n",
    "dropout = 0.2 # We use dropout\n",
    "\n",
    "# Use this in the CSV filename when saving\n",
    "# when using dropout\n",
    "filename_extension = ''\n",
    "if dropout>0.0:\n",
    "    filename_extension = '_dropout'\n",
    "    \n",
    "filename_to_save = 'lstm_word2vec'+filename_extension+'.csv' # use to save perplexity values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Inputs and Outputs\n",
    "\n",
    "In the code we define two different types of inputs. \n",
    "* Training inputs (The stories we downloaded) (batch_size > 1 with unrolling)\n",
    "* Validation inputs (An unseen validation dataset) (bach_size =1, no unrolling)\n",
    "* Test inputs (New story we are going to generate) (batch_size=1, no unrolling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Training Input data.\n",
    "train_inputs, train_labels = [],[]\n",
    "train_labels_ohe = []\n",
    "# Defining unrolled training inputs\n",
    "for ui in range(num_unrollings):\n",
    "    train_inputs.append(tf.placeholder(tf.int32, shape=[batch_size],name='train_inputs_%d'%ui))\n",
    "    train_labels.append(tf.placeholder(tf.int32, shape=[batch_size], name = 'train_labels_%d'%ui))\n",
    "    train_labels_ohe.append(tf.one_hot(train_labels[ui], vocabulary_size))\n",
    "    \n",
    "# Validation data placeholders\n",
    "valid_inputs = tf.placeholder(tf.int32, shape=[1],name='valid_inputs')\n",
    "valid_labels = tf.placeholder(tf.int32, shape=[1], name = 'valid_labels')\n",
    "valid_labels_ohe = tf.one_hot(valid_labels, vocabulary_size)\n",
    "\n",
    "# Text generation: batch 1, no unrolling.\n",
    "test_input = tf.placeholder(tf.int32, shape=[1],name='test_input')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Word Embeddings to TensorFlow\n",
    "We load the previously learned and stored embeddings to TensorFlow and define tensors to hold embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## If you want to change the embedding matrix to something you newly generated,\n",
    "## Simply change embeddings.npy to embeddings-tmp.npy\n",
    "embed_mat = np.load('embeddings.npy')\n",
    "embeddings_size = embed_mat.shape[1]\n",
    "\n",
    "embed_init = tf.constant(embed_mat)\n",
    "embeddings = tf.Variable(embed_init,name='embeddings')\n",
    "\n",
    "# Defining embedding lookup operations for all the unrolled\n",
    "# trianing inputs\n",
    "train_inputs_embeds = []\n",
    "for ui in range(num_unrollings):\n",
    "    # We use expand_dims to add an additional axis\n",
    "    # As this is needed later for LSTM cell computation\n",
    "    train_inputs_embeds.append(tf.expand_dims(tf.nn.embedding_lookup(embeddings,train_inputs[ui]),0))\n",
    "\n",
    "# Defining embedding lookup for operations for all the validation data\n",
    "valid_inputs_embeds = tf.nn.embedding_lookup(embeddings,valid_inputs)\n",
    "\n",
    "# Defining embedding lookup for operations for all the testing data\n",
    "test_input_embeds = tf.nn.embedding_lookup(embeddings, test_input)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Model Parameters\n",
    "\n",
    "Now we define model parameters. Compared to RNNs, LSTMs have a large number of parameters. Each gate (input, forget, memory and output) has three different sets of parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defining softmax weights and biases\n",
      "Defining the LSTM cell\n"
     ]
    }
   ],
   "source": [
    "print('Defining softmax weights and biases')\n",
    "# Softmax Classifier weights and biases.\n",
    "w = tf.Variable(tf.truncated_normal([num_nodes[-1], vocabulary_size], stddev=0.01))\n",
    "b = tf.Variable(tf.random_uniform([vocabulary_size],0.0,0.01))\n",
    "\n",
    "print('Defining the LSTM cell')\n",
    "# Defining a deep LSTM from Tensorflow RNN API\n",
    "\n",
    "# First we define a list of LSTM cells\n",
    "# num_nodes here is a sequence of hidden layer sizes\n",
    "cells = [tf.nn.rnn_cell.LSTMCell(n) for n in num_nodes]\n",
    "\n",
    "# We now define a dropout wrapper for each LSTM cell\n",
    "dropout_cells = [\n",
    "    rnn.DropoutWrapper(\n",
    "        cell=lstm, input_keep_prob=1.0,\n",
    "        output_keep_prob=1.0-dropout, state_keep_prob=1.0,\n",
    "        variational_recurrent=True, \n",
    "        input_size=tf.TensorShape([embeddings_size]),\n",
    "        dtype=tf.float32\n",
    "    ) for lstm in cells\n",
    "]\n",
    "\n",
    "# We first define a MultiRNNCell Object that uses the \n",
    "# Dropout wrapper (for training)\n",
    "stacked_dropout_cell = tf.nn.rnn_cell.MultiRNNCell(dropout_cells)\n",
    "# Here we define a MultiRNNCell that does not use dropout\n",
    "# Validation and Testing\n",
    "stacked_cell = tf.nn.rnn_cell.MultiRNNCell(cells)\n",
    "\n",
    "\n",
    "# Note: There exists the EmbeddingWrapper in RNN API to automate the embedding_lookup but,\n",
    "# in many cases it may be more efficient to not use this wrapper, but instead concatenate the whole sequence of \n",
    "# your inputs in time, do the embedding on this batch-concatenated sequence, then split it and feed into your RNN.\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining LSTM Computations\n",
    "Here first we define the LSTM cell computations as a consice function. Then we use this function to define training and test-time inference logic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LSTM calculations for unrolled inputs and outputs\n"
     ]
    }
   ],
   "source": [
    "print('LSTM calculations for unrolled inputs and outputs')\n",
    "# =========================================================\n",
    "# Training inference logic\n",
    "\n",
    "# Initial state of the LSTM memory.\n",
    "initial_state = stacked_dropout_cell.zero_state(batch_size, dtype=tf.float32)\n",
    "\n",
    "# Defining the LSTM cell computations (training)\n",
    "train_outputs, initial_state = tf.nn.dynamic_rnn(\n",
    "    stacked_dropout_cell, tf.concat(train_inputs_embeds,axis=0), \n",
    "    time_major=True, initial_state=initial_state\n",
    ")\n",
    "\n",
    "# Reshape the final outputs to [num_unrollings*batch_size, num_nodes]\n",
    "final_output = tf.reshape(train_outputs,[-1,num_nodes[-1]])\n",
    "\n",
    "# Computing logits\n",
    "logits = tf.matmul(final_output, w) + b\n",
    "# Computing predictions\n",
    "train_prediction = tf.nn.softmax(logits)\n",
    "\n",
    "# Reshape logits to time-major fashion [num_unrollings, batch_size, vocabulary_size]\n",
    "time_major_train_logits = tf.reshape(logits,[num_unrollings,batch_size,-1])\n",
    "\n",
    "# We create train labels in a time major fashion [num_unrollings, batch_size, vocabulary_size]\n",
    "# so that this could be used with the loss function\n",
    "time_major_train_labels = tf.reshape(tf.concat(train_labels,axis=0),[num_unrollings,batch_size])\n",
    "\n",
    "# Perplexity related operation\n",
    "train_perplexity_without_exp = tf.reduce_sum(tf.concat(train_labels_ohe,0)*-tf.log(train_prediction+1e-10))/(num_unrollings*batch_size)\n",
    "\n",
    "# =========================================================\n",
    "# Validation inference logic\n",
    "\n",
    "# Separate state for validation data\n",
    "initial_valid_state = stacked_cell.zero_state(1, dtype=tf.float32)\n",
    "\n",
    "# Validation input related LSTM computation\n",
    "valid_outputs, initial_valid_state = tf.nn.dynamic_rnn(\n",
    "    stacked_cell, tf.expand_dims(valid_inputs_embeds,0), \n",
    "    time_major=True, initial_state=initial_valid_state\n",
    ")\n",
    "\n",
    "# Reshape the final outputs to [1, num_nodes]\n",
    "final_valid_output = tf.reshape(valid_outputs,[-1,num_nodes[-1]])\n",
    "\n",
    "# Computing logits\n",
    "valid_logits = tf.matmul(final_valid_output, w) + b\n",
    "# Computing predictions\n",
    "valid_prediction = tf.nn.softmax(valid_logits)\n",
    "\n",
    "# Perplexity related operation\n",
    "valid_perplexity_without_exp = tf.reduce_sum(valid_labels_ohe*-tf.log(valid_prediction+1e-10))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LSTM Loss\n",
    "We calculate the training loss of the LSTM here. It's a typical cross entropy loss calculated over all the scores we obtained for training data (`loss`) and averaged and summed in a specific way."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# We use the sequence-to-sequence loss function to define the loss\n",
    "# We calculate the average across the batches\n",
    "# But get the sum across the sequence length\n",
    "loss = tf.contrib.seq2seq.sequence_loss(\n",
    "    logits = tf.transpose(time_major_train_logits,[1,0,2]),\n",
    "    targets = tf.transpose(time_major_train_labels),\n",
    "    weights= tf.ones([batch_size, num_unrollings], dtype=tf.float32),\n",
    "    average_across_timesteps=False,\n",
    "    average_across_batch=True\n",
    ")\n",
    "\n",
    "loss = tf.reduce_sum(loss)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Learning Rate and the Optimizer with Gradient Clipping\n",
    "Here we define the learning rate and the optimizer we're going to use. We will be using the Adam optimizer as it is one of the best optimizers out there. Furthermore we use gradient clipping to prevent any gradient explosions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Defining optimizer\n"
     ]
    }
   ],
   "source": [
    "# Used for decaying learning rate\n",
    "gstep = tf.Variable(0, trainable=False)\n",
    "\n",
    "# Running this operation will cause the value of gstep\n",
    "# to increase, while in turn reducing the learning rate\n",
    "inc_gstep = tf.assign(gstep, gstep+1)\n",
    "\n",
    "# Adam Optimizer. And gradient clipping.\n",
    "tf_learning_rate = tf.train.exponential_decay(0.001,gstep,decay_steps=1, decay_rate=0.5)\n",
    "\n",
    "print('Defining optimizer')\n",
    "optimizer = tf.train.AdamOptimizer(tf_learning_rate)\n",
    "gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "gradients, _ = tf.clip_by_global_norm(gradients, 5.0)\n",
    "optimizer = optimizer.apply_gradients(\n",
    "    zip(gradients, v))\n",
    "\n",
    "inc_gstep = tf.assign(gstep, gstep+1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM with Beam-Search\n",
    "\n",
    "Here we alter the previously defined prediction related TensorFlow operations to employ beam-search. Beam search is a way of predicting several time steps ahead. Concretely instead of predicting the best prediction we have at a given time step, we get predictions for several time steps and get the sequence of highest joint probability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 48]\n"
     ]
    }
   ],
   "source": [
    "beam_length = 5\n",
    "beam_neighbors = 5\n",
    "\n",
    "# We redefine the sample generation with beam search\n",
    "sample_beam_inputs = [tf.placeholder(tf.int32, shape=[1]) for _ in range(beam_neighbors)]\n",
    "# Embedding lookups for each beam\n",
    "sampel_beam_input_embeds = [tf.nn.embedding_lookup(embeddings,b) for b in sample_beam_inputs]\n",
    "\n",
    "best_beam_index = tf.placeholder(shape=None, dtype=tf.int32)\n",
    "best_neighbor_beam_indices = tf.placeholder(shape=[beam_neighbors], dtype=tf.int32)\n",
    "\n",
    "# We have [num_layers, beam_neighbors] shape state variable set\n",
    "# Maintains output of each beam\n",
    "saved_sample_beam_output = [[tf.Variable(tf.zeros([1, n])) for _ in range(beam_neighbors)] for n in num_nodes]\n",
    "# Maintains the state of each beam\n",
    "saved_sample_beam_state = [[tf.Variable(tf.zeros([1, n])) for _ in range(beam_neighbors)] for n in num_nodes] \n",
    "\n",
    "# Resetting the sample beam states (should be done at the beginning of each text snippet generation)\n",
    "reset_sample_beam_state = tf.group(\n",
    "    *[[saved_sample_beam_output[ni][vi].assign(tf.zeros([1, n]))  for vi in range(beam_neighbors)] for ni,n in enumerate(num_nodes)],\n",
    "    *[[saved_sample_beam_state[ni][vi].assign(tf.zeros([1, n])) for vi in range(beam_neighbors)] for ni,n in enumerate(num_nodes)] \n",
    ")\n",
    "\n",
    "# We stack them to perform gather operation below\n",
    "# These should be of size [beam_neighbors, 1, num_nodes]\n",
    "stacked_beam_outputs = [tf.stack(saved_sample_beam_output[n]) for n in range(len(num_nodes))]\n",
    "stacked_beam_states = [tf.stack(saved_sample_beam_state[n]) for n in range(len(num_nodes))]\n",
    "\n",
    "# The beam states for each beam (there are beam_neighbor-many beams) needs to be updated at every depth of tree\n",
    "# Consider an example where you have 3 classes where we get the best two neighbors (marked with star)\n",
    "#     a`      b*       c  \n",
    "#   / | \\   / | \\    / | \\\n",
    "#  a  b c  a* b` c  a  b  c\n",
    "# Since both the candidates from level 2 comes from the parent b\n",
    "# We need to update both states/outputs from saved_sample_beam_state/output to have index 1 (corresponding to parent b)\n",
    "\n",
    "# Our update_sample_beam_state gets very complicated\n",
    "# Because we have to do this for every beam neighbor \n",
    "# as well as every layer\n",
    "update_sample_beam_state = tf.group(\n",
    "    *[\n",
    "        [saved_sample_beam_output[n][vi].assign(\n",
    "            tf.gather_nd(stacked_beam_outputs[n],[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)\n",
    "          for n in range(len(num_nodes))]\n",
    "    ],\n",
    "    *[\n",
    "        [saved_sample_beam_state[n][vi].assign(\n",
    "            tf.gather_nd(stacked_beam_states[n],[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)\n",
    "          for n in range(len(num_nodes))]\n",
    "    ]\n",
    ")\n",
    "\n",
    "# This needs to be of shap [beam_neighbors, num_layers]\n",
    "sample_beam_outputs, sample_beam_states = [],[] \n",
    "\n",
    "# This needs to be of shape [beam_neighbors, num_layers]\n",
    "# and each item is a LSTMStateTuple\n",
    "# We calculate lstm_cell state and output for each beam\n",
    "tmp_state_tuple = []\n",
    "for vi in range(beam_neighbors):\n",
    "    single_beam_state_tuple = []\n",
    "    for ni in range(len(num_nodes)):\n",
    "        single_beam_state_tuple.append(\n",
    "            tf.nn.rnn_cell.LSTMStateTuple(saved_sample_beam_output[ni][vi], saved_sample_beam_state[ni][vi])\n",
    "        )\n",
    "    tmp_state_tuple.append(single_beam_state_tuple)\n",
    "\n",
    "\n",
    "for vi in range(beam_neighbors):\n",
    "    # We cannot use tf.nn.dynamic_rnn as we need to manipulate\n",
    "    # LSTM state a lot. So even though it is lot of work\n",
    "    # It is easier to do state manipulation externelly \n",
    "    # when using beam search\n",
    "    final_output, tmp_state_tuple[vi] = stacked_cell.call(\n",
    "        sampel_beam_input_embeds[vi], tmp_state_tuple[vi]\n",
    "    )\n",
    "    \n",
    "    # We need to be care how we populate sample_beam_outputs\n",
    "    # and sample_beam_state\n",
    "    # They both need to be of size [beam_neighbors, num_layers]\n",
    "    sample_beam_outputs.append([])\n",
    "    sample_beam_states.append([])\n",
    "    for ni in range(len(num_nodes)):\n",
    "        sample_beam_outputs[-1].append(tmp_state_tuple[vi][ni][0])\n",
    "        sample_beam_states[-1].append(tmp_state_tuple[vi][ni][1])\n",
    "\n",
    "    \n",
    "# This store predictions made for each beam neighbor position\n",
    "sample_beam_predictions = []\n",
    "\n",
    "# Used to update the LSTM cell for each neighbor \n",
    "# Just as normally we do during generation\n",
    "beam_update_ops = tf.group(\n",
    "    [[saved_sample_beam_output[ni][vi].assign(sample_beam_outputs[vi][ni]) for vi in range(beam_neighbors)]\n",
    "                            for ni in range(len(num_nodes))],\n",
    "    [[saved_sample_beam_state[ni][vi].assign(sample_beam_states[vi][ni]) for vi in range(beam_neighbors)]\n",
    "                            for ni in range(len(num_nodes))]\n",
    ")\n",
    "\n",
    "# Get the predictions out\n",
    "# For a given set of beams, outputs a list of prediction vectors of size beam_neighbors\n",
    "# each beam having the predictions for full vocabulary\n",
    "for vi in range(beam_neighbors):\n",
    "    with tf.control_dependencies([beam_update_ops]):\n",
    "        sample_beam_predictions.append(tf.nn.softmax(tf.nn.xw_plus_b(sample_beam_outputs[vi][-1], w, b)))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM + Word2vec with Beam-Search\n",
    "\n",
    "Here we alter the previously defined prediction related TensorFlow operations to employ beam-search. Beam search is a way of predicting several time steps ahead. Concretely instead of predicting the best prediction we have at a given time step, we get predictions for several time steps and get the sequence of highest joint probability."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning rate Decay Logic\n",
    "\n",
    "Here we define the logic to decrease learning rate whenever the validation perplexity does not decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning rate decay related\n",
    "# If valid perpelxity does not decrease\n",
    "# continuously for this many epochs\n",
    "# decrease the learning rate\n",
    "decay_threshold = 5\n",
    "# Keep counting perplexity increases\n",
    "decay_count = 0\n",
    "\n",
    "min_perplexity = 1e10\n",
    "\n",
    "# Learning rate decay logic\n",
    "def decay_learning_rate(session, v_perplexity):\n",
    "  global decay_threshold, decay_count, min_perplexity  \n",
    "  # Decay learning rate\n",
    "  if v_perplexity < min_perplexity:\n",
    "    decay_count = 0\n",
    "    min_perplexity= v_perplexity\n",
    "  else:\n",
    "    decay_count += 1\n",
    "\n",
    "  if decay_count >= decay_threshold:\n",
    "    print('\\t Reducing learning rate')\n",
    "    decay_count = 0\n",
    "    session.run(inc_gstep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the Beam Prediction Logic\n",
    "Here we define function that takes in the session as an argument and output a beam of predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_word = None\n",
    "\n",
    "def get_beam_prediction(session):\n",
    "    \n",
    "    global test_word\n",
    "    global sample_beam_predictions, update_sample_beam_state\n",
    "    \n",
    "    # Generating words within a segment with Beam Search\n",
    "    # To make some calculations clearer, we use the example as follows\n",
    "    # We have three classes with beam_neighbors=2 (best candidate denoted by *, second best candidate denoted by `)\n",
    "    # For simplicity we assume best candidate always have probability of 0.5 in output prediction\n",
    "    # second best has 0.2 output prediction\n",
    "    #           a`                   b*                   c                <--- root level\n",
    "    #    /     |     \\         /     |     \\        /     |     \\   \n",
    "    #   a      b      c       a*     b`     c      a      b      c         <--- depth 1\n",
    "    # / | \\  / | \\  / | \\   / | \\  / | \\  / | \\  / | \\  / | \\  / | \\\n",
    "    # a b c  a b c  a b c   a*b c  a`b c  a b c  a b c  a b c  a b c       <--- depth 2\n",
    "    # So the best beams at depth 2 would be\n",
    "    # b-a-a and b-b-a\n",
    "\n",
    "    \n",
    "\n",
    "    # Calculate the candidates at the root level\n",
    "    feed_dict = {}\n",
    "    for b_n_i in range(beam_neighbors):\n",
    "        feed_dict.update({sample_beam_inputs[b_n_i]: [test_word]})\n",
    "\n",
    "    # We calculate sample predictions for all neighbors with the same starting word/character\n",
    "    # This is important to update the state for all instances of beam search\n",
    "    sample_preds_root = session.run(sample_beam_predictions, feed_dict = feed_dict)  \n",
    "    sample_preds_root = sample_preds_root[0]\n",
    "\n",
    "    # indices of top-k candidates\n",
    "    # b and a in our example (root level)\n",
    "    this_level_candidates_sorted =  (np.argsort(sample_preds_root,axis=1).ravel()[::-1]).tolist() # indices of top-k candidates\n",
    "    this_level_candidates = []\n",
    "    for c in this_level_candidates_sorted:\n",
    "        if len(this_level_candidates)==beam_neighbors:\n",
    "            break\n",
    "        if c!=0:\n",
    "            this_level_candidates.append(c)\n",
    "\n",
    "    this_level_candidates = np.array(this_level_candidates)\n",
    "\n",
    "    # probabilities of top-k candidates\n",
    "    # 0.5 and 0.2\n",
    "    this_level_probs = sample_preds_root[0,this_level_candidates] #probabilities of top-k candidates\n",
    "\n",
    "    # Update test sequence produced by each beam from the root level calculation\n",
    "    # Test sequence looks like for our example (at root)\n",
    "    # [b,a]\n",
    "    test_sequences = ['' for _ in range(beam_neighbors)]\n",
    "    for b_n_i in range(beam_neighbors):\n",
    "        test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]] + ' '\n",
    "\n",
    "    # Make the calculations for the rest of the depth of the beam search tree\n",
    "    for b_i in range(beam_length-1):\n",
    "        test_words = [] # candidate words for each beam\n",
    "        pred_words = [] # Predicted words of each beam\n",
    "\n",
    "        # computing feed_dict for the beam search (except root)\n",
    "        # feed dict should contain the best words/chars/bigrams found by the previous level of search\n",
    "\n",
    "        # For level 1 in our example this would be\n",
    "        # sample_beam_inputs[0]: b, sample_beam_inputs[1]:a\n",
    "        feed_dict = {}\n",
    "        for p_idx, pred_i in enumerate(this_level_candidates):                    \n",
    "            # Updating the feed_dict for getting next predictions\n",
    "            test_words.append(this_level_candidates[p_idx])\n",
    "\n",
    "            feed_dict.update({sample_beam_inputs[p_idx]:[test_words[p_idx]]})\n",
    "\n",
    "        # Calculating predictions for all neighbors in beams\n",
    "        # This is a list of vectors where each vector is the prediction vector for a certain beam\n",
    "        # For level 1 in our example, the prediction values for \n",
    "        #      b             a  (previous beam search results)\n",
    "        # [a,  b,  c],  [a,  b,  c] (current level predictions) would be\n",
    "        # [0.1,0.1,0.1],[0.5,0.2,0]\n",
    "        sample_preds_all_neighbors = session.run(sample_beam_predictions, feed_dict=feed_dict)\n",
    "\n",
    "        # Create a single vector with \n",
    "        # Making our example [0.1,0.1,0.1,0.5,0.2,0] \n",
    "        sample_preds_all_neighbors_concat = np.concatenate(sample_preds_all_neighbors,axis=1)\n",
    "\n",
    "        # Update this_level_candidates to be used for the next iteration\n",
    "        # And update the probabilities for each beam\n",
    "        # In our example these would be [3,4] (indices with maximum value from above vector)\n",
    "        # We also use a simple trick to avoid UNK (word id 0) being predicted \n",
    "        this_level_candidates_sorted = np.argsort(sample_preds_all_neighbors_concat.ravel())[::-1]\n",
    "        this_level_candidates = []\n",
    "        for c in this_level_candidates_sorted:\n",
    "            if len(this_level_candidates)==beam_neighbors:\n",
    "                break\n",
    "            if c!=0 and c%vocabulary_size != 0 :\n",
    "                this_level_candidates.append(c)\n",
    "\n",
    "        this_level_candidates = np.array(this_level_candidates)\n",
    "\n",
    "        # In the example this would be [1,1]\n",
    "        parent_beam_indices = this_level_candidates//vocabulary_size\n",
    "\n",
    "        # normalize this_level_candidates to fall between [0,vocabulary_size]\n",
    "        # In this example this would be [0,1]\n",
    "        this_level_candidates = (this_level_candidates%vocabulary_size).tolist()\n",
    "\n",
    "        # Here we update the final state of each beam to be\n",
    "        # the state that was at the index 1. Because for both the candidates at this level the parent is \n",
    "        # at index 1 (that is b from root level)\n",
    "        session.run(update_sample_beam_state, feed_dict={best_neighbor_beam_indices: parent_beam_indices})\n",
    "\n",
    "        # Here we update the joint probabilities of each beam and add the newly found candidates to the sequence\n",
    "        tmp_this_level_probs = np.asarray(this_level_probs)\n",
    "        tmp_test_sequences = list(test_sequences)\n",
    "\n",
    "        for b_n_i in range(beam_neighbors):\n",
    "            # We make the b_n_i element of this_level_probs to be the probability of parents\n",
    "            # In the example the parent indices are [1,1]\n",
    "            # So this_level_probs become [0.5,0.5]\n",
    "            this_level_probs[b_n_i] = tmp_this_level_probs[parent_beam_indices[b_n_i]]\n",
    "\n",
    "            # Next we multipyle these by the probabilities of the best candidates from current level \n",
    "            # [0.5*0.5, 0.5*0.2] = [0.25,0.1]\n",
    "            this_level_probs[b_n_i] *= sample_preds_all_neighbors[parent_beam_indices[b_n_i]][0,this_level_candidates[b_n_i]]\n",
    "\n",
    "            # Make the b_n_i element of test_sequences to be the correct parent of the current best candidates\n",
    "            # In the example this becomes [b, b]\n",
    "            test_sequences[b_n_i] = tmp_test_sequences[parent_beam_indices[b_n_i]]\n",
    "\n",
    "            # Now we append the current best candidates\n",
    "            # In this example this becomes [ba,bb]\n",
    "            test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]] + ' '\n",
    "\n",
    "            # Create one-hot-encoded representation for each candidate\n",
    "            pred_words.append(this_level_candidates[b_n_i])\n",
    "\n",
    "    # Calculate best beam id based on the highest beam probability\n",
    "    # Using the highest beam probability always lead to very monotonic text\n",
    "    # Let us sample one randomly where one being sampled is decided by the likelihood of that beam\n",
    "    rand_cand_ids = np.argsort(this_level_probs)[-3:]\n",
    "    rand_cand_probs = this_level_probs[rand_cand_ids]/np.sum(this_level_probs[rand_cand_ids])\n",
    "    random_id = np.random.choice(rand_cand_ids,p=rand_cand_probs)\n",
    "\n",
    "    best_beam_id = parent_beam_indices[random_id]\n",
    "    # Update state and output variables for test prediction\n",
    "    session.run(update_sample_beam_state,feed_dict={best_neighbor_beam_indices:[best_beam_id for _ in range(beam_neighbors)]})\n",
    "\n",
    "    # Make the last word/character/bigram from the best beam\n",
    "    test_word = pred_words[best_beam_id]\n",
    "        \n",
    "    return test_sequences[best_beam_id]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running Training, Validation and Generation\n",
    "\n",
    "We traing the LSTM on existing training data, check the validaiton perplexity on an unseen chunk of text and generate a fresh segment of text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "num_steps = 251\n",
    "steps_per_document = 100\n",
    "docs_per_step = 10\n",
    "valid_summary = 1\n",
    "train_doc_count = 100"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Training (Step: 0)\n",
      "(1).(62).(82).(67).(57).(87).(58).(10).(30).(6).\n",
      "Average loss at step 1: 5.133673\n",
      "\tPerplexity at step 1: 169.639030\n",
      "\n",
      "Valid Perplexity: 232.29\n",
      "\n",
      "Generated Text after epoch 0 ... \n",
      "======================== New text Segment ==========================\n",
      " UNK her , , , ,  , , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  and , , , ,  , , , , ,  . , , , ,  her , , , ,  her , , , ,  and , , , ,  the , , , ,  the , , , ,  and , , , ,  and , , , ,  and , , , ,  . , , , ,  . , , , ,  and , , , ,  and , , , ,  her , , , ,  , , , , ,  . , , , ,  the , , , ,  . , , , ,  the , , , ,  the , , , ,  , , , , ,  . , , , ,  the , , , ,  . , , , ,  and , , , ,  and , , , ,  her , , , ,  the , , , ,  the , , , ,  . , , , ,  and , , , ,  , , , , ,  . , , , ,  , , , , ,  . , , , ,  . , , , ,  the , , , ,  and , , , ,  and , , , ,  the , , , ,  . , , , ,  the , , , ,  her , , , ,  the , , , ,  the , , , ,  her , , , ,  the , , , ,  the , , , ,  and , , , ,  , , , , ,  , , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  . , , , ,  her , , , ,  the , , , ,  the , , , ,  , , , , ,  , , , , ,  the , , , ,  the , , , ,  , , , , ,  and , , , ,  the , , , ,  . , , , ,  and , , , ,  the , , , ,  her , , , ,  , , , , ,  , , , , ,  and , , , ,  her , , , ,  , , , , ,  and , , , ,  and , , , ,  her , , , ,  and , , , ,  , , , , ,  her , , , ,  the , , , ,  the , , , ,  the , , , ,  her , , , ,  the , , , ,  and , , , ,  the , , , ,  , , , , ,  and , , , ,  and , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 1)\n",
      "(72).(20).(11).(49).(31).(94).(64).(65).(35).(85).\n",
      "Average loss at step 2: 5.085651\n",
      "\tPerplexity at step 2: 161.685143\n",
      "\n",
      "Valid Perplexity: 195.28\n",
      "\n",
      "Generated Text after epoch 1 ... \n",
      "======================== New text Segment ==========================\n",
      " father , , , , ,  , , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  , , , , ,  to , , , ,  , , , , ,  the , , , ,  , , , , ,  the , , , ,  , , , , ,  , , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  the , , , ,  . , , , ,  the , , , ,  to , , , ,  the , , , ,  the , , , ,  to , , , ,  , , , , ,  the , , , ,  , , , , ,  the , , , ,  the , , , ,  , , , , ,  to , , , ,  and , , , ,  to , , , ,  the , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  . , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  the , , , ,  the , , , ,  . , , , ,  to , , , ,  the , , , ,  and , , , ,  . , , , ,  the , , , ,  and , , , ,  to , , , ,  to , , , ,  the , , , ,  . , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  the , , , ,  and , , , ,  the , , , ,  the , , , ,  the , , , ,  , , , , ,  the , , , ,  to , , , ,  . , , , ,  the , , , ,  to , , , ,  to , , , ,  the , , , ,  to , , , ,  . , , , ,  the , , , ,  the , , , ,  , , , , ,  to , , , ,  the , , , ,  . , , , ,  and , , , ,  the , , , ,  , , , , ,  and , , , ,  , , , , ,  to , , , ,  , , , , ,  , , , , ,  , , , , ,  and , , , ,  , , , , ,  and , , , ,  the , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 2)\n",
      "(82).(10).(47).(46).(7).(66).(71).(77).(24).(0).\n",
      "Average loss at step 3: 5.052397\n",
      "\tPerplexity at step 3: 156.396863\n",
      "\n",
      "Valid Perplexity: 190.89\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 2 ... \n",
      "======================== New text Segment ==========================\n",
      " to , , , , ,  , , , , ,  , , , , ,  and , , , ,  . , , , ,  to , , , ,  and , , , ,  . , , , ,  the , , , ,  . , , , ,  to , , , ,  and , , , ,  , , , , ,  to , , , ,  , , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  the , , , ,  the , , , ,  and , , , ,  and , , , ,  the , , , ,  , , , , ,  and , , , ,  the , , , ,  and , , , ,  and , , , ,  , , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  and , , , ,  and , , , ,  to , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  and , , , ,  and , , , ,  to , , , ,  , , , , ,  and , , , ,  and , , , ,  . , , , ,  . , , , ,  and , , , ,  , , , , ,  . , , , ,  , , , , ,  to , , , ,  and , , , ,  and , , , ,  . , , , ,  , , , , ,  and , , , ,  and , , , ,  and , , , ,  . , , , ,  the , , , ,  and , , , ,  , , , , ,  and , , , ,  . , , , ,  and , , , ,  to , , , ,  and , , , ,  and , , , ,  the , , , ,  and , , , ,  and , , , ,  and , , , ,  and , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 3)\n",
      "(70).(28).(24).(5).(64).(20).(76).(40).(23).(78).\n",
      "Average loss at step 4: 4.964746\n",
      "\tPerplexity at step 4: 143.272116\n",
      "\n",
      "Valid Perplexity: 210.09\n",
      "\n",
      "Generated Text after epoch 3 ... \n",
      "======================== New text Segment ==========================\n",
      " his , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 4)\n",
      "(24).(0).(36).(47).(82).(76).(50).(88).(13).(74).\n",
      "Average loss at step 5: 4.816891\n",
      "\tPerplexity at step 5: 123.580301\n",
      "\n",
      "Valid Perplexity: 183.25\n",
      "\n",
      "Generated Text after epoch 4 ... \n",
      "======================== New text Segment ==========================\n",
      " her , , , , ,  and , , , ,  . , , , ,  . , , , ,  to , , , ,  and , , , ,  to , , , ,  . , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  and , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  the , , , ,  . , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 5)\n",
      "(58).(33).(34).(10).(57).(8).(83).(2).(36).(96).\n",
      "Average loss at step 6: 4.677417\n",
      "\tPerplexity at step 6: 107.492090\n",
      "\n",
      "Valid Perplexity: 156.75\n",
      "\n",
      "Generated Text after epoch 5 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " . when and , , ,  to , , , ,  . , , , ,  in , , , ,  in , , , ,  in , , , ,  . , , , ,  , , , , ,  . , , , ,  in , , , ,  and , , , ,  in , , , ,  and , , , ,  in , , , ,  , , , , ,  . , , , ,  in , , , ,  , , , , ,  , , , , ,  , , , , ,  in , , , ,  and , , , ,  , , , , ,  in , , , ,  in , , , ,  . , , , ,  in , , , ,  . , , , ,  in , , , ,  . , , , ,  , , , , ,  in , , , ,  . , , , ,  . , , , ,  , , , , ,  . , , , ,  . , , , ,  , , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,  . , , , ,  in , , , ,  , , , , ,  in , , , ,  . , , , ,  . , , , ,  in , , , ,  , , , , ,  , , , , ,  . , , , ,  in , , , ,  , , , , ,  in , , , ,  in , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  . , , , ,  , , , , ,  . , , , ,  . , , , ,  in , , , ,  , , , , ,  in , , , ,  , , , , ,  . , , , ,  in , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  in , , , ,  in , , , ,  in , , , ,  , , , , ,  in , , , ,  , , , , ,  . , , , ,  , , , , ,  in , , , ,  . , , , ,  , , , , ,  . , , , ,  in , , , ,  in , , , ,  . , , , ,  . , , , ,  in , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 6)\n",
      "(34).(63).(27).(56).(87).(68).(94).(66).(84).(42).\n",
      "Average loss at step 7: 4.496319\n",
      "\tPerplexity at step 7: 89.686410\n",
      "\n",
      "Valid Perplexity: 141.44\n",
      "\n",
      "Generated Text after epoch 6 ... \n",
      "======================== New text Segment ==========================\n",
      " son , , , , ,  to , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,  . , , , ,  , , , , ,  . , , , ,  . , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 7)\n",
      "(18).(78).(70).(69).(87).(22).(53).(89).(5).(84).\n",
      "Average loss at step 8: 4.305322\n",
      "\tPerplexity at step 8: 74.093089\n",
      "\n",
      "Valid Perplexity: 123.78\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 7 ... \n",
      "======================== New text Segment ==========================\n",
      " longer he , , , ,  . , , , ,  . , , , ,  . , , , ,  to , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  , , , , ,  to , , , ,  to , , , ,  . , , , ,  of , , , ,  of , , , ,  . , , , ,  , , , , ,  . , , , ,  in , , , ,  to , , , ,  . , , , ,  to , , , ,  , , , , ,  . , , , ,  of , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  of , , , ,  of , , , ,  of , , , ,  with , , , ,  of , , , ,  of , , , ,  with , , , ,  of , , , ,  of , , , ,  of , , , ,  of , , , ,  with , , , ,  with , , , ,  of , , , ,  of , , , ,  of , , , ,  with , , , ,  of , , , ,  with , , , ,  with , , , ,  , , , , ,  of , , , ,  of , , , ,  with , , , ,  with , , , ,  with , , , ,  of , , , ,  , , , , ,  with , , , ,  of , , , ,  to , , , ,  , , , , ,  of , , , ,  with , , , ,  with , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  . , , , ,  . , , , ,  of , , , ,  , , , , ,  of , , , ,  to , , , ,  of , , , ,  . , , , ,  . , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 8)\n",
      "(94).(83).(82).(62).(9).(17).(96).(69).(44).(0).\n",
      "Average loss at step 9: 4.252020\n",
      "\tPerplexity at step 9: 70.247166\n",
      "\n",
      "Valid Perplexity: 118.09\n",
      "\n",
      "Generated Text after epoch 8 ... \n",
      "======================== New text Segment ==========================\n",
      " the king , , , ,  to , , , ,  and , , , ,  of , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  into , , , ,  , , , , ,  . , , , ,  . , , , ,  . , , , ,  of , , , ,  of , , , ,  . , , , ,  , , , , ,  , , , , ,  to , , , ,  into , , , ,  into , , , ,  of , , , ,  , , , , ,  of , , , ,  . , , , ,  of , , , ,  , , , , ,  of , , , ,  . , , , ,  to , , , ,  of , , , ,  of , , , ,  of , , , ,  of , , , ,  , , , , ,  , , , , ,  . , , , ,  to , , , ,  , , , , ,  to , , , ,  , , , , ,  to , , , ,  of , , , ,  . , , , ,  to , , , ,  , , , , ,  of , , , ,  of , , , ,  to , , , ,  , , , , ,  to , , , ,  to , , , ,  of , , , ,  to , , , ,  to , , , ,  of , , , ,  of , , , ,  of , , , ,  , , , , ,  to , , , ,  . , , , ,  , , , , ,  , , , , ,  of , , , ,  . , , , ,  , , , , ,  of , , , ,  . , , , ,  . , , , ,  of , , , ,  , , , , ,  of , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 9)\n",
      "(52).(64).(62).(94).(14).(29).(39).(34).(42).(3).\n",
      "Average loss at step 10: 4.172191\n",
      "\tPerplexity at step 10: 64.857385\n",
      "\n",
      "Valid Perplexity: 118.60\n",
      "\n",
      "Generated Text after epoch 9 ... \n",
      "======================== New text Segment ==========================\n",
      " `` , and and , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,  , , , , ,  . , , , ,  of , , , ,  to , , , ,  . , , , ,  of , , , ,  to , , , ,  to , , , ,  , , , , ,  , , , , ,  , , , , ,  in , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  of , , , ,  to , , , ,  , , , , ,  . , , , ,  . , , , ,  in , , , ,  to , , , ,  in , , , ,  of , , , ,  , , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  of , , , ,  , , , , ,  . , , , ,  , , , , ,  of , , , ,  . , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  . , , , ,  of , , , ,  . , , , ,  . , , , ,  to , , , ,  . , , , ,  of , , , ,  of , , , ,  , , , , ,  of , , , ,  . , , , ,  to , , , ,  . , , , ,  's , , , ,  to , , , ,  of , , , ,  of , , , ,  of , , , ,  . , , , ,  to , , , ,  of , , , ,  to , , , ,  , , , , ,  of , , , ,  . , , , ,  of , , , ,  . , , , ,  of , , , ,  . , , , ,  . , , , ,  of , , , ,  of , , , ,  . , , , ,  . , , , ,  . , , , ,  of , , , ,  . , , , ,  of , , , ,  of , , , ,  of , , , ,  of , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  of , , , ,  of , , , ,  . , , , ,  . , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 10)\n",
      "(78).(82).(84).(35).(31).(15).(0).(77).(11).(76).\n",
      "Average loss at step 11: 4.129361\n",
      "\tPerplexity at step 11: 62.138221\n",
      "\n",
      "Valid Perplexity: 111.39\n",
      "\n",
      "Generated Text after epoch 10 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " , , and and , ,  , , , , ,  to , , , ,  , , , , ,  , , , , ,  to , , , ,  , , , , ,  's , , , ,  , , , , ,  's , , , ,  's , , , ,  . , , , ,  , , , , ,  to , , , ,  to , , , ,  to , , , ,  . , , , ,  of , , , ,  , , , , ,  of , , , ,  of , , , ,  to , , , ,  to , , , ,  to , , , ,  to , , , ,  . , , , ,  of , , , ,  , , , , ,  , , , , ,  . , , , ,  's , , , ,  to , , , ,  to , , , ,  , , , , ,  . , , , ,  's , , , ,  of , , , ,  's , , , ,  of , , , ,  's , , , ,  to , , , ,  , , , , ,  , , , , ,  to , , , ,  's , , , ,  , , , , ,  to , , , ,  . , , , ,  , , , , ,  of , , , ,  to , , , ,  . , , , ,  . , , , ,  to , , , ,  to , , , ,  . , , , ,  of , , , ,  of , , , ,  of , , , ,  of , , , ,  . , , , ,  . , , , ,  of , , , ,  . , , , ,  , , , , ,  . , , , ,  . , , , ,  's , , , ,  of , , , ,  , , , , ,  . , , , ,  . , , , ,  . , , , ,  's , , , ,  . , , , ,  , , , , ,  's , , , ,  , , , , ,  of , , , ,  of , , , ,  . , , , ,  . , , , ,  . , , , ,  , , , , ,  's , , , ,  's , , , ,  . , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  's , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  . , , , ,  of , , , ,  's , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 11)\n",
      "(4).(57).(7).(34).(75).(31).(35).(36).(78).(69).\n",
      "Average loss at step 12: 4.017763\n",
      "\tPerplexity at step 12: 55.576666\n",
      "\n",
      "Valid Perplexity: 106.26\n",
      "\n",
      "Generated Text after epoch 11 ... \n",
      "======================== New text Segment ==========================\n",
      " and said , , , ,  that , , , ,  to , , , ,  of , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  of , , , ,  , , , , ,  , , , , ,  of , , , ,  on , , , ,  , , , , ,  in , , , ,  , , , , ,  , , , , ,  in , , , ,  , , , , ,  of , , , ,  , , , , ,  to , , , ,  to , , , ,  in , , , ,  to , , , ,  in , , , ,  of , , , ,  in , , , ,  . , , , ,  of , , , ,  , , , , ,  in , , , ,  . , , , ,  , , , , ,  . , , , ,  in , , , ,  . , , , ,  . , , , ,  , , , , ,  in , , , ,  of , , , ,  to , , , ,  , , , , ,  , , , , ,  to , , , ,  , , , , ,  . , , , ,  in , , , ,  in , , , ,  of , , , ,  , , , , ,  of , , , ,  , , , , ,  of , , , ,  to , , , ,  . , , , ,  of , , , ,  , , , , ,  in , , , ,  of , , , ,  , , , , ,  in , , , ,  to , , , ,  . , , , ,  , , , , ,  to , , , ,  of , , , ,  , , , , ,  of , , , ,  to , , , ,  to , , , ,  to , , , ,  , , , , ,  to , , , ,  , , , , ,  in , , , ,  to , , , ,  . , , , ,  of , , , ,  . , , , ,  to , , , ,  . , , , ,  . , , , ,  , , , , ,  , , , , ,  in , , , ,  , , , , ,  in , , , ,  to , , , ,  of , , , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,  in , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 12)\n",
      "(75).(90).(1).(20).(67).(2).(92).(53).(65).(64).\n",
      "Average loss at step 13: 4.011054\n",
      "\tPerplexity at step 13: 55.205047\n",
      "\n",
      "Valid Perplexity: 97.79\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 12 ... \n",
      "======================== New text Segment ==========================\n",
      " who went and , , ,  and , , , ,  for , , , ,  that , , , ,  that , , , ,  that , , , ,  that , , , ,  that , , , ,  that , , , ,  that , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  to , , , ,  , , , , ,  . , , , ,  , , , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  to , , , ,  , , , , ,  . , , , ,  . , , , ,  in , , , ,  . , , , ,  . , , , ,  . , , , ,  . , , , ,  of , , , ,  in , , , ,  to , , , ,  of , , , ,  in , , , ,  of , , , ,  . , , , ,  , , , , ,  , , , , ,  in , , , ,  in , , , ,  , , , , ,  in , , , ,  of , , , ,  . , , , ,  in , , , ,  of , , , ,  . , , , ,  , , , , ,  to , , , ,  . , , , ,  , , , , ,  . , , , ,  to , , , ,  to , , , ,  in , , , ,  to , , , ,  . , , , ,  of , , , ,  to , , , ,  . , , , ,  to , , , ,  in , , , ,  , , , , ,  . , , , ,  . , , , ,  to , , , ,  , , , , ,  . , , , ,  in , , , ,  to , , , ,  . , , , ,  in , , , ,  in , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 13)\n",
      "(75).(76).(13).(70).(7).(67).(23).(35).(89).(3).\n",
      "Average loss at step 14: 3.826662\n",
      "\tPerplexity at step 14: 45.909040\n",
      "\n",
      "Valid Perplexity: 100.80\n",
      "\n",
      "Generated Text after epoch 13 ... \n",
      "======================== New text Segment ==========================\n",
      " that he , and , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , that  at in in that that  with in in in in  "
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\ipykernel_launcher.py:140: RuntimeWarning: invalid value encountered in true_divide\n",
      "c:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\ipykernel_launcher.py:141: RuntimeWarning: invalid value encountered in less\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "that in in in in  that in in in in  to in in in in  with in in in in  that in in in in  that that that in in  with in in in in  that in in in in  that in in in in  that in in in in  that in in in in  that in in in in  to in in in in  that in in in in  that in in in in  that in in in in  to in in in in  that in in in in  that in in in in  that in in in in  to in in in in  that in in in in  that in in in in  that in in in in  to in in in in  that in in in in  that in in in in  that in in in in  to in in in in  that in in in in  that in in in in  that in in in in  in in in in in  with in in in in  that in in in in  that in in in in  that in in in in  in in in in in  to in in in in  to in in in in  , in in in in  , in in in in  that in in in in  that in in in in  in in in in in  that in in in in  , in in in in  that in in in in  with in in in in  , in in in in  to in in in in  with in in in in  that in in in in  that in in in in  in in in in in  that in in in in  , in in in in  in in in in in  , in in in in  , in in in in  , in in in in  in in in in in  , in in in in  that in in in in  that in in in in  , in in in in  that in in in in  , in in in in  , in in in in  , in in in in  , in in in in  , in in in in  that in in in in  in in in in in  with in in in in  , in in in in  to in in in in  that in in in in  in in in in in  , in in in in  , in in in in  to in in in in  , in in in in  with in in in in   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 14)\n",
      "(31).(89).(14).(47).(59).(63).(82).(29).(35).(21).\n",
      "Average loss at step 15: 3.675874\n",
      "\tPerplexity at step 15: 39.483141\n",
      "\n",
      "Valid Perplexity: 92.69\n",
      "\n",
      "Generated Text after epoch 14 ... \n",
      "======================== New text Segment ==========================\n",
      " on you , , , ,  , he , and he  , and that that that  in in and and the  that in it and and  in the and and and  and and and and and  that and in and the  the and and and and  that and in in the  the and and and and  that and and in in  it and and and and  in that and and and  in and and and and  that he and and in  in the and and and  that and and and and  in the and and and  and and and and the  one and and and and  in that and and and  in the and and and  that and and and and  in the and and and  in and and and and  you and and in you  in the and and and  and and and and and  he he he and and  in that and in and  it and the and with  with and and and the  one he and and and  that and in and and  in and and that that  that that and and that  that that that that the  that in that and and  that that that that that  that that that that and  in that and and that  that that that that that  in that that that that  in that that that that  in that that that that  in that that that that  in that that that that  in that that that that  it in that that in  in that that that in  that and and that that  in that that in and  so and that that that  in that and and that  that in that that and  and in that and and  in that that that the  so and in and and  that that that that that  he and and in in  it and and and and  in and and and and  in and and and and  in you and and the  the and and and in  in the and and and  and and and and and  he he he and and  in that and that and  in and that that that  that that and and that  that that that that that  in that and and that  that that that that that  that that that that the  that and in that that  with in and and and  that that that that the  the and and in that  in that and and that  that that that that and  that that that that that  and that that in in  that and and in that  that and and that that  in in the and so  in and and and and  that that that that the  that in and and and  that that that that that  that that that that and  in that that and and  that that that that the  that in that and and  that in that and and  in that that and and  that that in that and  in that and and and  that that that that and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 15)\n",
      "(7).(66).(85).(67).(6).(30).(60).(41).(36).(61).\n",
      "Average loss at step 16: 4.009716\n",
      "\tPerplexity at step 16: 55.131204\n",
      "\n",
      "Valid Perplexity: 92.79\n",
      "\n",
      "Generated Text after epoch 15 ... \n",
      "======================== New text Segment ==========================\n",
      " a time , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 16)\n",
      "(26).(70).(14).(85).(57).(92).(58).(10).(50).(95).\n",
      "Average loss at step 17: 3.818192\n",
      "\tPerplexity at step 17: 45.521851\n",
      "\n",
      "Valid Perplexity: 87.80\n",
      "\n",
      "Generated Text after epoch 16 ... \n",
      "======================== New text Segment ==========================\n",
      " reward and that that that that  , that that that that  that that that that that  on that that that that  it that that that that  that that that that that  that that that that that  on that that that that  it that that that that  to that that that that  on that that that that  , that that that that  of that that that that  of that that that that  that that that that that  that that that that that  that that that that that  to that that that that  on that that that that  on that that that that  on that that that that  that that that that that  , that that that that  that that that that that  on that that that that  on that that that that  that that that that that  to that that that that  on on that that that  to that that that that  that that that that that  on that that that that  that that that that that  on that that that that  that that that that that  to that that that that  on on that that that  to that that that that  that that that that that  on that that that that  that that that that that  , that that that that  of that that that that  on that that that that  to that that that that  on on that that that  that that that that that  on that that that that  that that that that that  to that that that that  on that that that that  on that that that that  that that that that that  to that that that that  of that that that that  , that that that that  to that that that that  to that that that that  that that that that that  to that that that that  to that that that that  that that that that that  , that that that that  , that that that that  to that that that that  on on that that that  to that that that that  on that that that that  on that that that that  that that that that that  of that that that that  on that that that that  to that that that that  , that that that that  on that that that that  on that that that that  of that that that that  that that that that that  to that that that that  on that that that that  on that that that that  that that that that that  to that that that that  to that that that that  to that that that that  to that that that that  , that that that that  to that that that that  in that that that that  that that that that that  on that that that that  , that that that that  that that that that that  on that that that that  on that that that that  , that that that that  of that that that that  of that that that that  that that that that that  on that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 17)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(6).(62).(22).(55).(65).(11).(82).(16).(79).(50).\n",
      "Average loss at step 18: 3.871798\n",
      "\tPerplexity at step 18: 48.028668\n",
      "\n",
      "Valid Perplexity: 87.65\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 17 ... \n",
      "======================== New text Segment ==========================\n",
      " , , and that that that  that that that that that  as that that that that  that that that that that  that that that that that  that that that that that  as that that that that  as that that that that  as that that that that  so that that that that  in that that that that  it that that that that  so that that that that  that that that that that  that that that that that  so that that that that  that that that that that  in that that that that  so that that that that  so that that that that  so that that that that  that that that that that  so that that that that  in that that that that  that that that that that  so that that that that  that that that that that  that that that that that  in that that that that  that that that that that  in that that that that  that that that that that  so that that that that  that that that that that  in that that that that  that that that that that  so that that that that  so that that that that  as that that that that  that that that that that  as that that that that  , that that that that  that that that that that  that that that that that  that that that that that  in that that that that  so that that that that  in that that that that  that that that that that  in that that that that  as that that that that  in that that that that  in that that that that  in that that that that  in that that that that  , that that that that  so that that that that  so that that that that  so that that that that  as that that that that  in that that that that  as that that that that  so that that that that  as that that that that  so that that that that  on that that that that  on that that that that  , that that that that  that that that that that  that that that that that  in that that that that  that that that that that  in that that that that  that that that that that  in that that that that  as that that that that  that that that that that  so that that that that  in that that that that  as that that that that  in that that that that  that that that that that  so that that that that  that that that that that  that that that that that  as that that that that  that that that that that  as that that that that  that that that that that  , that that that that  so that that that that  as that that that that  so that that that that  , that that that that  that that that that that  that that that that that  that that that that that  as that that that that  that that that that that  as that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 18)\n",
      "(94).(27).(56).(15).(55).(3).(14).(87).(81).(83).\n",
      "Average loss at step 19: 3.756922\n",
      "\tPerplexity at step 19: 42.816446\n",
      "\n",
      "Valid Perplexity: 93.02\n",
      "\n",
      "Generated Text after epoch 18 ... \n",
      "======================== New text Segment ==========================\n",
      " time and in it in it  , and in in the  in that and in in  that in it in it  it and in in the  which and in in in  it and in in and  it and in in and  and and in in the  that in in in the  one and in in in  it and and in in  that and in it in  it and and in in  that and in in the  in it in it and  and in in in the  in it and in in  and in it and in  that and in it in  in the and and in  it in in it and  it and in in and  it and in in the  which and in in in  it and in in and  that and in in the  in that in in the  in it and in in  it and in it and  it in in it and  that and in in in  that and in it in  in it and and in  that and in it in  and in it and in  that and in it in  and in it and in  in it and and in  it in it and and  and in in in it  in it and in in  it and and in in  that and in it in  it and in in it  it and and in in  that in it in it  that and in in it  in it and in in  in the and and in  that in in it in  in it and and in  it in it and in  that and in it in  and in it and in  and in in the and  that in in it and  in it in it and  and in in in the  one and in in in  that and in in the  one in it and in  it in it and in  in and and in in  in the and and in  it in it and in  it and in in and  that and in in in  it and and in in  it and in in it  in it and and in  it in it and in  that and in in it  in it and and in  that in it in it  it in it and in  in and and in in  in it and and in  it in it and in  that and in it in  in it and and in  and in it and in  in it and and in  that in it in it  it and in in it  and and in in it  in it and in in  the and and in in  that in it in it  in it and in in  it and and in in  in the and and in  that in in it and  in it in it and  and in in in the  in it and in in  that in it and in  it in it and in  it and in in and  it and in in and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 19)\n",
      "(51).(66).(54).(59).(71).(5).(45).(40).(80).(8).\n",
      "Average loss at step 20: 3.850917\n",
      "\tPerplexity at step 20: 47.036170\n",
      "\n",
      "Valid Perplexity: 83.82\n",
      "\n",
      "Generated Text after epoch 19 ... \n",
      "======================== New text Segment ==========================\n",
      " at him , and that the  , , that that ,  it , that that that  that that that that that  as that that that that  in that that that that  as that that that that  that that that that that  that , that that that  it , that , that  that that that that that  as that that that that  in that that that that  as that that that that  in that that that that  that that that that that  as that that that that  in that that that that  that that that that that  as that that that that  in that that that that  as that that that that  in that that that that  , that , that that  as that that that that  of that that that that  that that that that that  that that that that that  that that that that that  in that that that that  , that , that that  as that that that that  in that that that that  it that that that that  that that that that that  that , that that that  it , that , that  that that that that that  as that that that that  in that that that that  as that that that that  in that that that that  , that , that that  , that that , that  as that that , that  , that that that that  that that that that that  as that that that that  in that that that that  , that , that that  , that that , that  as that that , that  , that that that that  as that that that that  of that that that that  that that that that that  as that that that that  in that that that that  it that that that that  that , that , that  , that that that that  , that that that that  as that that that that  in that that that that  it that that that that  , that , that that  to that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  in that that that that  , that , that that  to that that that that  , that , that that  , that , that that  , that that , that  that that that that that  , that , that that  as that that that that  , that that that that  that , that that that  that that that that that  that that that that that  that that that that that  in that that that that  as that that that that  in that that that that  it that that that that  that , that , that  as that that that that  that that that that that  that , that that that  , that , that that  , that that that that  that that that that that  , that that that that  , that , that that  , that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 20)\n",
      "(22).(21).(9).(17).(30).(96).(50).(45).(29).(12).\n",
      "Average loss at step 21: 3.735418\n",
      "\tPerplexity at step 21: 41.905542\n",
      "\n",
      "Valid Perplexity: 86.57\n",
      "\n",
      "Generated Text after epoch 20 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " UNK . and that that the  that that that that that  it that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  , that that that that  that that that that that  that that that that that  a that that that that  that that that that that  that that that that that  a that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  , that that that that  a that that that that  that that that that that  that that that that that  it that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  it that that that that  that that that that that  , that that that that  the that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  it that that that that  that that that that that  it that that that that  that that that that that  that that that that that  , that that that that  the that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  it that that that that  the that that that that  that that that that that  that that that that that  it that that that that  that that that that that  that that that that that  that that that that that  it that that that that  that that that that that  it that that that that  that that that that that  that that that that that  it that that that that  that that that that that  that that that that that  it that that that that  that that that that that  that that that that that  it that that that that  that that that that that  that that that that that  it that that that that  that that that that that  it that that that that  that that that that that  that that that that that  that that that that that  , that that that that  a that that that that  that that that that that  it that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  it that that that that  , that that that that  the that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 21)\n",
      "(23).(98).(64).(74).(27).(9).(20).(66).(13).(18).\n",
      "Average loss at step 22: 3.802451\n",
      "\tPerplexity at step 22: 44.810885\n",
      "\n",
      "Valid Perplexity: 84.78\n",
      "\n",
      "Generated Text after epoch 21 ... \n",
      "======================== New text Segment ==========================\n",
      " who said , you come and  to you for and a  one and and and the  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 22)\n",
      "(98).(84).(58).(59).(96).(52).(94).(80).(37).(83).\n",
      "Average loss at step 23: 3.528847\n",
      "\tPerplexity at step 23: 34.084651\n",
      "\n",
      "Valid Perplexity: 88.38\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 22 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " human silver , and and and  she , that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 23)\n",
      "(5).(38).(12).(79).(8).(1).(40).(76).(42).(48).\n",
      "Average loss at step 24: 3.754396\n",
      "\tPerplexity at step 24: 42.708434\n",
      "\n",
      "Valid Perplexity: 83.87\n",
      "\n",
      "Generated Text after epoch 23 ... \n",
      "======================== New text Segment ==========================\n",
      " . but and that that that  that that that that that  that that that that the  that that that that that  as that that that that  that that that that the  that that that that that  as that that that that  as that that that that  as that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that that  that that that that the  that that that that that  that that that that the  that that that that that  as that that that that  as that that that that  as that that that that  that that that that the  that that that that that  that that that that the  that that that that that  as that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that that  as that that that that  that that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that that  that that that that the  that that that that that  as that that that that  that that that that the  that that that that that  that that that that that  that that that that the  one that that that that  as that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that that  that that that that the  that that that that that  that that that that the  one that that that that  as that that that that  that that that that the  that that that that that  that that that that that  that that that that the  one that that that that  as that that that that  that that that that the  that that that that that  that that that that the  that that that that that  as that that that that  as that that that that  that that that that that  as that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that that  that that that that the  one that that that that  as that that that that  that that that that the  that that that that that  that that that that the  that that that that that  that that that that that  that that that that the  that that that that that  that that that that the  that that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 24)\n",
      "(97).(39).(17).(33).(59).(73).(49).(43).(92).(5).\n",
      "Average loss at step 25: 3.629567\n",
      "\tPerplexity at step 25: 37.696500\n",
      "\n",
      "Valid Perplexity: 79.29\n",
      "\n",
      "Generated Text after epoch 24 ... \n",
      "======================== New text Segment ==========================\n",
      " miller said , i i i  i i i i i  i i i i i  i i i i i  i i i i i  i i i i i  no there there as that  that as as , i  that that that i is  there and as as that  that as that as that  , the one , that  that that that that that  that that that that that  as the that , the  man , and that the  , said that , the  one , he , that  , , that , i  i i i i i  i i i i i  i i i i i  i i i i i  no there there there there  as that as that the  day as that , the  man , and that the  man , and that that  , he , that that  that that that that that  as that , the one  that that , that that  , the one , the  man , and he and  that that that that that  that the that , he  that that that that that  as the that , the  man , and that the  , said that that the  that , he , the  that , he , that  , what that that the  that , he he the  one , and that the  man , and he the  at that that , he  , and that that the  that that that that that  that that that that that  as the that , the  man , and that the  , said that that the  that , that , he  , that , that that  as that as that as  that that that that ,  he , that that the  that that that that the  that , that , he  that that that that that  as the that , the  that , he , and  and that that that the  , that that that that  that that that that that  , the that , he  that that that he the  that , he , and  and that , that the  that that that that the  that , that , he  that that that that that  as the that , the  man , and that the  , said that , the  one , he , that  he , that , that  what that that as that  as that as that that  that that that that that  as the that , the  man , and that the  , said that that the  that , that , i  , you i i i  i i i i i  i i i i i  i i i i i  i i i i i  no there there there there  that that that as that  as that as that that  that that that that ,  she , that that that  it , that , the  day that that , the  man , and that the  , said that that the  that , that , i   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 25)\n",
      "(69).(31).(30).(88).(56).(40).(38).(22).(57).(58).\n",
      "Average loss at step 26: 3.453076\n",
      "\tPerplexity at step 26: 31.597438\n",
      "\n",
      "Valid Perplexity: 83.52\n",
      "\n",
      "Generated Text after epoch 25 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " . and he he and to  that and that the that  that that and and and  at that that and and  and and at that and  and the that and at  that and and and at  that that and and at  that that and and at  that that and and the  at that and and at  that that and and the  at that and and the  that and and and at  that that and and and  that that and that that  and that that that that  that that the that and  , that , and the  at that that that that  that that that that and  and at that and and  and at that that and  , the and and the  that and and at the  at that , he and  and and and and at  that that and and at  that that and and at  that that and and at  that and and at out  he to that and to  and at out and the  at out and and the  at that and and and  and at that that and  and at that and that  and and and at the  at that that and and  and and and that that  and that that that that  that that the that and  , that that and the  at that that to the  at that and at the  one and and in the  table that and and to  at at out and to  that when that to the  at that , and to  that and and and and  at that that and the  at that and and the  at that and and at  that that and and at  that that and and the  at that and and and  and and that that the  one and at and and  and at that that and  , the that and and  and and at that the  that , he , and  to to that and when  he to to to a  wife to the at for  that and and and and  at that that and the  at that and and at  that that and and the  that and and at the  at that , and and  and and and and at  that that and and at  that that and and the  at that and and the  that and and at the  at that , he and  and at and and the  that that and and at  that that and and the  at that and and the  and and and at the  that , and and at  that that and to to  that and and to to  at at out and at  that and and at at  that and and and and  that that and that that  that that and and at  that and that the that  that and and at that  and and and at the  at that , and to  that and and and and  at that that and the  at that and and the  at that and and at  that and and and and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 26)\n",
      "(49).(66).(6).(88).(86).(90).(80).(22).(97).(68).\n",
      "Average loss at step 27: 3.647987\n",
      "\tPerplexity at step 27: 38.397300\n",
      "\n",
      "Valid Perplexity: 82.12\n",
      "\n",
      "Generated Text after epoch 26 ... \n",
      "======================== New text Segment ==========================\n",
      " one . he he and the  in and and and that  and that that that that  and that that that that  that and that that that  , he , that that  that that that that that  that that that that that  and that that that that  , and and that that  , that that that that  that that that that that  that that that that that  , he , that that  , that that that that  , he , and and  at that that that that  , that that that that  , that that that that  that that that that that  that that that that that  and that that that that  , that that that that  , and and and and  and that that that that  that that that that that  , that that that that  , and that that that  that that that that that  , he , and and  and and that that that  , that that that that  , and and and and  that that that that that  that that that that that  he , that that that  , that that that that  , and that that that  that that that that that  and that that that that  , and that that that  , and that that that  , and that that that  that that that that that  and that that that that  , that that that that  that that that that that  and that that that that  , and and that that  and that that that that  , he , that that  , that that that that  , he , and and  at that that that that  , that that that that  , that that that that  that that that that that  and that that that that  , that that that that  , he , and and  and and that that that  , that that that that  , and and and and  that that that that that  , that that that that  , and that that that  , that that that that  , that that that that  that that that that that  and that that that that  that that that that that  , that that that that  , and that that that  , that that that that  that that that that that  that that that that that  that that that that that  and that that that that  , that that that that  , he , and and  on and and that that  and that that that that  that that that that that  and that that that that  he , that that that  that that that that that  and that that that that  , and and that that  , and and and and  that that that that that  that that that that that  he , that that that  , and that that that  , that that that that  that that that that that  that that that that that  that that that that that  and that that that that  , that that that that  that that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 27)\n",
      "(10).(43).(17).(68).(14).(54).(77).(44).(49).(47).\n",
      "Average loss at step 28: 3.581129\n",
      "\tPerplexity at step 28: 35.914080\n",
      "\n",
      "Valid Perplexity: 89.89\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 27 ... \n",
      "======================== New text Segment ==========================\n",
      " sent out , he and at  and and and the other  , which and and and  at on and and at  that and he , at  all and , and that  that that that that that  he that that that that  , that , the one  , which , who ,  that , that , the  child and and and at  at the one which that  he that that that that  he that that that that  , he , that the  that , he , at  on and he and the  one and and and and  and at that that that  that that that the one  , he , and at  and and he , who  who to to to to  and at on and the  one and and and at  that that , he ,  he , who to to  at at on and the  one and and and the  that , and at the  one and and and that  that that that that that  he that that that that  that that that that that  he that that the one  , he , and the  on and and and the  one and and and and  and at that that that  , he , the one  and and and and at  that that , he ,  he , who and to  and at on and at  and and that that at  that that he , at  that , he and to  and at on and the  one and and and the  that , and and the  on and and and at  that and that that that  that that that the one  , he to to to  what at all and the  that , and at which  he , who , and  , and , and the  which and , and and  which that that that that  , that , that that  that that at the one  that he that that the  one , he to to  at at on and the  one and and that that  , he to to to  at at one and the  king and and and and  and at that that ,  , and and and at  that that he , the  that and at all the  that and at all the  man , and and on  and and and at that  , and and and and  and at at the one  that that he , at  all and , and the  that and that that that  , he , the other  which which , who ,  but and and and at  out and and at on  that that he , and  that that that , he  , at all that ,  that that that that that  , he , that the  that , that at the  father who , and and  and and at at the  on that that that that  that that that the one  , he , who ,  but and and and on  and and and at that  , and , and and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 28)\n",
      "(82).(21).(87).(7).(99).(70).(52).(22).(33).(23).\n",
      "Average loss at step 29: 3.581400\n",
      "\tPerplexity at step 29: 35.923788\n",
      "\n",
      "Valid Perplexity: 89.70\n",
      "\n",
      "Generated Text after epoch 28 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " . hans that he , the  , and , , he  . , he , ,  and , , , ,  that , , , ,  he , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  was , , , ,  was . , , ,  . , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  was , , , ,  i , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  was , , , ,  i , , , ,  , , , , ,  , , , , ,  that , , , ,  he , , , ,  , , , , ,  , , , , ,  , , , , ,  i , , , ,  he , , , ,  i , , , ,  that , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  was , , , ,  i , , , ,  , , , , ,  , , , , ,  was , , , ,  i , , , ,  that , , , ,  , , , , ,  , , , , ,  that , , , ,  , , , , ,  he , , , ,  , , , , ,  that , , , ,  , , , , ,  , , , , ,  that , , , ,  i , , , ,  , , , , ,  , , , , ,  i , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  that , , , ,  he , , , ,  , , , , ,  , , , , ,  that , , , ,  he , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  , , , , ,  i , , , ,  he , , , ,  that , , , ,  he , , , ,  , , , , ,  he , , , ,  , , , , ,  that , , , ,  , , , , ,  , , , , ,  that , , , ,  he , , , ,  , , , , ,  , , , , ,  , , , , ,  i , , , ,  he , , , ,  he , , , ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 29)\n",
      "(46).(6).(32).(55).(59).(79).(82).(2).(30).(65).\n",
      "Average loss at step 30: 3.602449\n",
      "\tPerplexity at step 30: 36.687963\n",
      "\n",
      "Valid Perplexity: 79.94\n",
      "\n",
      "Generated Text after epoch 29 ... \n",
      "======================== New text Segment ==========================\n",
      " , he and and and and  which that that that that  , he , and and  and the one and the  one and and and the  and that that that that  that that that that the  what that , that ,  that that , he ,  he and there that the  that , and he ,  he and and and and  the that that that that  , he , and the  out and that that that  that that that that that  , he , and the  that and and and the  that that that that the  in which that that that  , he , and the  that , and and the  home and and and and  and and and that that  that that that that that  that that that that the  that that that that the  in that , he ,  there , that , and  that , that , i  i i i i i  i there i i there  there . when he the  there , and and the  man , and and on  her which that that ,  he and and and and  and that that that that  , he , that the  what that , that and  and and and which the  and and that that that  , he , and the  what that , that ,  he and there that the  that , and he and  and and and and that  and that that that that  there that that that the  that that that that that  , that , he ,  he and and and and  and that that that that  that that that that that  that that that that the  what that that that the  , and and and and  and that that that that  that that that that the  that that that that that  he , that , and  he , and and and  and the that that that  and that that that that  that that that that the  what that that that the  , and and and the  that and that that that  that that that that the  that that that that the  what that , that and  and and and that the  that , that that that  , he , and and  that that that that the  in that , he ,  there , that , and  that , that , i  i i i i i  there i i i there  there , , and and  there there there and the  one home and to the  and and and and the  that that that that that  , he , and the  man who , that ,  , he , and and  there that that , and  , and that that the  godfather and and and and  and and that that that  , he , and the  what that , that ,  he and there that that  , he and and the  man and and and and  and that that that that  that that that that that  that that that that the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 30)\n",
      "(71).(43).(90).(75).(60).(4).(64).(49).(37).(25).\n",
      "Average loss at step 31: 3.775381\n",
      "\tPerplexity at step 31: 43.614106\n",
      "\n",
      "Valid Perplexity: 79.51\n",
      "\n",
      "Generated Text after epoch 30 ... \n",
      "======================== New text Segment ==========================\n",
      " who said , i is not  you and i i i  do i , i will  have you . and she  , and he and not  that , and and the  night and she and and  and the one and the  one and and that that  that he , and the  little man and and the  and and and and the  and and and and that  and that that that he  , she and and and  and and that that she  , she and and and  and the one and and  and and the one and  and that that that he  , she and and and  and and that that she  , she and and and  and the one and and  and and the one and  and that that that he  , she and and and  and that that that the  good i will not be  there for the one and  i have you . and  he and her one and  that that that he the  one , and she and  and and and that that  that she that that the  little man , and the  one , and he and  and and and and that  one as the one who  , she , and and  and and that that that  , she , and and  and that that that she  , and she and the  little and and and at  it and she and that  that she , she and  to not not but but  when she and at the  water and and and that  and she that that the  father and not not the  one for her the man  , she and that the  king and and and the  man , and that the  good man , she and  she and to not the  so and and that that  , she , she and  to not but but but  as that as the one  time , said she ,  if i not not not  you i shall not i  or not or not and  have you and i i  not i i i i  then i i i i  but i i i i  but i i i i  but i i i i  i i i i i  i i i i i  but i i no or  here or no or the  one which i do and  , and i not be  there or not and i  i i i i i  then i i i i  i i i i i  so i i i i  but i i i i  or no or i or  you and i i do  you and i i i  i i i i i  then i i i i  i i i i i  i i i no or  i or or or and  if i do and i  will not do and i  will not do and i  do not i i have  you , and i not  have you . and she  , she , and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 31)\n",
      "(33).(34).(37).(73).(51).(36).(65).(88).(28).(4).\n",
      "Average loss at step 32: 3.531562\n",
      "\tPerplexity at step 32: 34.177314\n",
      "\n",
      "Valid Perplexity: 76.61\n",
      "\n",
      "Generated Text after epoch 31 ... \n",
      "======================== New text Segment ==========================\n",
      " she said , i i do  i i i i i  i i i i i  as as i i i  i i i i i  as i i i i  as as i i i  i i i i i  if i i i i  i as as i i  i i as as i  i i i i i  as i i i i  i i i i i  as as i i i  i i i i i  as as i i i  as i i i i  i i i i i  i as as i i  i i i i i  as i i i i  if i i i i  i as as i i  i i as as i  i i i i i  i i i i i  as as i i i  as i i i i  i as as i i  i i i i i  as as i i i  i i i i i  if i i i i  i as as i i  i i as as i  i i i i i  as i i i i  at i i i i  if i i i i  as as i i i  at i i i i  i i i i i  i as as i i  if i i i i  as i i i as  if i i i i  i i i i i  as as i i i  i i i i i  as as i i i  i i i i i  as as i i i  i i i i i  as as i i i  as i i i i  if i i i i  i as as i i  i i as as i  i i i i i  i i i i i  i i i i i  i i as as i  i i i i i  i i i i i  as as i i i  as i i i i  as as i i i  i i i i i  as as i i i  i i i i i  as as i i i  i i i i i  as as i i i  i i i i i  as as i i i  as i i as as  as i i i i  i i i i i  as i i i i  as i i i i  if i i i i  i i i i i  if i i i i  i as as i i  i i as i i  if i i i i  as i i i i  as as i i i  if i i i i  as i i as as  i i i i i  i i i i i  as as i i i  i i i i i  as as i i i  as i i i i  if i i i i  as as i i i  if i i i i   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 32)\n",
      "(30).(61).(6).(86).(82).(69).(66).(81).(72).(24).\n",
      "Average loss at step 33: 3.540223\n",
      "\tPerplexity at step 33: 34.474606\n",
      "\n",
      "Valid Perplexity: 82.01\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 32 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " servant , and he and on  and and that that that  that that that that that  that that that that the  time that that that that  that that that that the  time that he that that  that that that the one  that that that that he  that that that that that  that that that the one  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that that  that that that that the  time that that that that  that that that that he  that that that that one  time as he he he  that that that that that  that he that that that  that that that that that  as the time that the  night , and he he  he that that that he  he that that that the  time that he that the  good and that that he  that that that that he  that that that that that  that that that the one  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that that  that that that one one  that that that that that  that that that that the  time that he that that  that that that the one  that that that that he  that that that that that  that that that that one  there as the one and  which that that that that  that that that that that  that that that that the  time that that that that  that that that the one  that that that as he  that that that that he  he that that the one  that that that that that  that that that that that  that that he one one  on him and and that  that it as he the  that that that that that  that that that that he  that that that the one  that that that that that  that that that that that  that that he one one  on him and and that  that that as the one  that that that that that  that that that that the  night that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that that  that that that that the  time that that that that  that that that that he  that that that that one  time as he he he  that that that that that  that that that the one  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he  that that that that he   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 33)\n",
      "(58).(89).(18).(2).(24).(8).(17).(19).(94).(41).\n",
      "Average loss at step 34: 3.490400\n",
      "\tPerplexity at step 34: 32.799065\n",
      "\n",
      "Valid Perplexity: 79.82\n",
      "\n",
      "Generated Text after epoch 33 ... \n",
      "======================== New text Segment ==========================\n",
      " had no little and and the  mother and and that the  one which that that that  that that that that that  that that she that that  she that that that that  she that that that that  she that that that that  that that that that that  as she that that that  he that that that that  that that she that that  that that that that that  that that that that that  she that that that that  she that that that the  night , and she and  and so that that that  she she that that that  she that that that that  she that that that that  she that that that that  she that that that the  night , and she and  and and that that that  he that that that that  he that that that that  she that that that that  she that that that the  night , and she the  little son and and and  that that that that that  that that that that the  night that that she that  so that that that that  that that that that that  as she that that that  he that that that that  that that she that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  that that that that that  as she that that that  he that that that that  that that she that that  she that that that that  he that that that that  she that that that that  that that that that that  as she that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  that that that that that  that that that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  she that that that that  that that that that that  as she that that that  he that that that that  that that she that that  she that that that that  he that that that that  she that that that that  that that that that that  as she that that that  he that that that that  that that she that that  she that that that that  he that that that that  she that that that that  that that that that that  as she that that that  he that that that that  he that that that the  mother that that she the  mother , and and and  that that that that that  he that that that that  he that that that that  he that that that that  that that that that that  as she that that that  he that that that that  that that she that that  she that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 34)\n",
      "(16).(56).(27).(33).(53).(74).(57).(38).(77).(67).\n",
      "Average loss at step 35: 3.441402\n",
      "\tPerplexity at step 35: 31.230721\n",
      "\n",
      "Valid Perplexity: 84.91\n",
      "\n",
      "Generated Text after epoch 34 ... \n",
      "======================== New text Segment ==========================\n",
      " called and and he he the  one and that that that  and he that that that  he that that that that  he that that that that  there that that that that  that that that that that  that that that that that  it that that that that  he that that that that  it that that that that  he that that that that  he that that that that  he that that that that  that that that that that  he that that that that  that that that that that  it that that that that  it that that that that  he that that that that  it that that that that  it that that that that  he that that that that  that that that that that  he that that that that  he that that that that  that that that that that  that that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  that that that that that  that that that that that  he that that that that  he that that that that  that that that that that  that that that that that  that that that that that  he that that that that  that that that that that  he that that that that  he that that that that  that that that that that  it that that that that  it that that that that  that that that that that  he that that that that  that that that that that  it that that that that  it that that that that  he that that that that  he that that that that  he that that that that  that that that that that  he that that that that  he that that that that  he that that that that  that that that that that  it that that that that  it that that that that  he that that that that  he that that that that  that that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  he that that that that  that that that that that  he that that that that  it that that that that  he that that that that  he that that that that  that that that that that  he that that that that  that that that that that  that that that that that  he that that that that  he that that that that  it that that that that  he that that that that  that that that that that  it that that that that  it that that that that  he that that that that  it that that that that  it that that that that  he that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 35)\n",
      "(72).(74).(96).(24).(30).(4).(49).(28).(16).(68).\n",
      "Average loss at step 36: 3.321062\n",
      "\tPerplexity at step 36: 27.689741\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Valid Perplexity: 81.77\n",
      "\n",
      "Generated Text after epoch 35 ... \n",
      "======================== New text Segment ==========================\n",
      " him , and he and that  and that that that he  , that that he and  on it and and and  and that that that that  that that that that that  he that that that she  he that that that that  she and that that he  , and that that that  she and that that that  that that that that he  he that that that that  she and that that the  one , that that that  that that that that that  he that that that that  he that that that he  , that and and the  one and which that that  and and that that he  that that that that the  that that that that the  that that that that that  he that that that the  man , that that that  he that that he and  that that that that he  he that that that that  he that that that he  he that that that that  she and that that he  , that and and that  he that that that he  that that that that that  that that that that he  he that that that he  , that that he and  all that that that that  that that that that the  that that that that that  he that that that the  man , and that that  he that that that the  one , that , and  that as he and the  that that that that the  father , and he and  all that that that that  it and that that he  , that that that he  , that , and that  as he that that that  he that it that the  that that that that that  he that that that the  one , that that the  father , that that the  man , who , the  man and that that he  that that that that that  that that that that that  and and that that the  that that that that he  that that that he and  all that that that he  he that that that that  that that that that the  that that that that that  he and that that the  that that he that the  other who that that and  that that that that he  he that that that he  he that that that that  she and that that he  , that and and that  he that that that he  , that that that that  that that that that the  that that that that that  he that that that that  he that that that that  that that that that he  he that that that he  , that that that he  he that that that that  she and that that he  , and that that that  that that that that he  he that that that the  man , that that that  she that that that that  he that that that he  he that that that he  who that that that that  she and that that that  that that that that he  he that that that that  she and that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 36)\n",
      "(65).(84).(93).(76).(63).(55).(88).(68).(72).(32).\n",
      "Average loss at step 37: 3.512405\n",
      "\tPerplexity at step 37: 33.528812\n",
      "\n",
      "Valid Perplexity: 83.88\n",
      "\n",
      "Generated Text after epoch 36 ... \n",
      "======================== New text Segment ==========================\n",
      " there , and he and all  that that that that the  father would not go home  . the father and and  and and and and that  and that that that that  he that that that the  king and and and one  of that that that that  that that that that that  that that the one has  not the one for you  that that that i will  go for you and the  man for it for the  people and that that you  i you that that you  i i i i i  i i i you you  to so so that the  father would go home and  i i i i i  so so one i will  go home and the life  . he and that that  that that that that he  that that that the little  son was and and the  son and and and and  and that that that that  that that that that that  that that the one has  all so one and and  so one and the one  in the life and a  so and and and that  that that that that he  he that that that that  that that that that he  that that that the little  old king and which that  it that the little daughter  , that the little king  , said that that that  that that that that the  father would not go home  . the father and and  and and and and that  that that that that that  he that that that he  that that that the little  son was and and the  son and and and the  and one and that that  he that that that the  king and and and one  of them that that he  , and that that that  that that that that he  , that that that one  as that that that he  that that that the little  old king and and the  son and and and and  one and that that that  he that that that the  father and and and that  , that that that that  it that that that he  that that that that that  that that the one has  all so one and but  that that that you will  go and and it for  that and the little daughter  , and he that the  father and that the little  one would not go out  and and that that that  that that the one has  all so one and but  that that the one who  , you will not you  that you i i will  to so that you will  do it and and that  that that that that i  i i i i i  so so one i will  go home and the one  , and i will not  so and and the one  of it . the little  man was and and the  other and and that that  it that that that the  father and and that that  that that that that that  that that that the one  who that that that you   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 37)\n",
      "(38).(87).(55).(50).(79).(58).(56).(32).(78).(81).\n",
      "Average loss at step 38: 3.313541\n",
      "\tPerplexity at step 38: 27.482264\n",
      "\n",
      "Valid Perplexity: 81.42\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 37 ... \n",
      "======================== New text Segment ==========================\n",
      " that i will not not you  so and and the one  you , and i have  it so and for the  father , and i will  not not you for it  , and i i no  in your life for your  father , and i have  it and you that the  good i has you the  wife , and i have  you that you i have  it and you you and  in the water which the  wife you have it the  wife , and he was  there for all and and  and so one and that  that that the little man  i you i i have  if you i i i  i as you if if  you i i as as  as as as as as  as as as as as  as as as as as  that and and he the  good , and that that  that that that that and  said , i have the  one for the good one  , i will not not  so one for it the  one and you you you  if you i i as  as as as as as  as as as as the  father would have the good  life for you , and  you i do you and  for the one which i  i i i i as  as as as as as  as as as as as  and and and he he  that that that he the  one in the table the  death which the little man  came and and he the  that and that that the  man , and that he  , that i have the  one who you you you  you i i i as  i i as if i  one as as as as  as as as and as  that , he , the  mother , and he that  that that that that that  the little king has not  that you i i have  you that and i have  it so and and the  father , he will not  be so one and the  one for it and the  one for it for the  one , and i have  it so and for the  king , and i have  it so and for the  father , and i will  not not you for it  , and i i no  in your life for your  one , and i will  you you for it .  if i do the good  i i i i have  you the one and you  i i i i as  as as as as as  as as as as the  father was for his little  house and and the one  that the little man has  all for it and the  one for it for it  , i i have no  good , and i will  not you for it for  it . i do you  for you for you can  have it for you ,  as i i have ,  and i will not not  you for it for your  one for it and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 38)\n",
      "(92).(69).(5).(18).(14).(36).(55).(90).(53).(52).\n",
      "Average loss at step 39: 3.436189\n",
      "\tPerplexity at step 39: 31.068324\n",
      "\n",
      "Valid Perplexity: 80.54\n",
      "\n",
      "Generated Text after epoch 38 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " hunger . the king and and  which and and and and  and that that that that  that that the one and  and so so so the  little tailor was and and  and and and and that  he that that that the  dear king , he and  and and and and and  and that that that that  that that the father ,  that you can not so  so and the father and  that the one and who  , the other and and  and so one and the  dear son , and he  , and that that that  that that that that that  that that that that the  good , he was so  little and and that that  that that that that that  i you you you you  i you you you you  you you you you you  you you you no to  to no so so that  that i you you no  one , you can not  you so so and when  that he that that the  dear king , the father  were and and and and  and that that that that  that that that the one  , who is to take  you so and and the  one home and be so  so that the king was  and and and and and  and one and and and  and that that that that  that that the father is  . that i are so  as the father is not  no and be and and  and so one and the  one and and and the  good home , and the  one and and and and  so one and that that  a little tailor , he  he and and and and  so that that that the  death , and the little  king and and and the  maiden and and and and  other and that that that  that that the one and  and and and so one  so so that he the  father who that that that  that that that the one  , and you can no  and and and so one  and that the little tailor  , and that the little  king , and he the  little son , and and  the other and and the  one and and and and  so one and that the  man , and the father  's daughter , and the  king , and the other  that that he that the  one and and so so  that the father was so  and and and that he  he that the other and  that that that that the  man , who he the  son , and he that  so that and and all  all so and and that  he that that that the  father , and he the  one and and and and  but so one and the  dear son , and the  king , and and all  that the one and who  , the other and and  that that that he the  good and and so so  and the one and the  little son , and he  that that that that the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 39)\n",
      "(61).(13).(46).(4).(1).(80).(3).(92).(41).(10).\n",
      "Average loss at step 40: 3.647942\n",
      "\tPerplexity at step 40: 38.395561\n",
      "\n",
      "Valid Perplexity: 77.36\n",
      "\n",
      "Generated Text after epoch 39 ... \n",
      "======================== New text Segment ==========================\n",
      " his queen , and she the  little son and all and  and the one and and  and that that that the  one , and that the  good woman , who is  for you , and you  will not not my one  with your life . the  mother and and and the  other and that that that  that that the one and  that that as she the  good , and she she  she the one and and  and that that that that  that that that that the  the queen , and you  will not not you and  and so one and the  good one and you you  not you you you you  as you do . she  and the one and and  all the one and which  she that that that the  beautiful son which that that  it so the one and  you you you you you  do not you you you  the one . and as  that as she the one  that the third which the  dear queen , who and  the little king and the  dear queen , and she  she that that that that  it the one and the  one home and do my  good one and you you  do not so so as  you do , and and  that that that the good  one and you you you  you you you you you  you you you you you  no as you will not  not not you for your  life . and she and  all the one and which  and that that that a  one which that that that  that you you you you  you you as you you  you you as you do  , and and as the  good boar , i do  you you you you you  you you you you you  you you you you you  you you you you you  my here you will not  not not you for you  , and you do not  so one for the good  one , and you have  not you you you you  you you you you you  you you you you you  you you here you do  as you do , and  as she do that that  i do not my one  in your life and the  three queen , and the  king , and that that  that that it my one  , and you do not  not so my one and  for you will not my  three and and the one  , and you have the  good one and you do  you you you you my  one for you will not  you you you you you  you you you you you  you you you you you  you here you do the  mother and do not my  one and and the one  , and you have the  good one and you will  not so so and the  king , and she the  one , and that that  she that it the one  , and that that as  you do that you can  do it and do not   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 40)\n",
      "(66).(84).(83).(11).(14).(92).(31).(56).(64).(50).\n",
      "Average loss at step 41: 3.344453\n",
      "\tPerplexity at step 41: 28.345062\n",
      "\n",
      "Valid Perplexity: 78.91\n",
      "\n",
      "Generated Text after epoch 40 ... \n",
      "======================== New text Segment ==========================\n",
      " . he and that that the  night that that she that  that that that that it  for it , and and  that that that that the  night that that he that  that that that that the  night , she and the  one which that that that  that that that that the  night that that that that  that that that that the  night , the king and  and and and that that  he that that that the  that that that she that  that that that that the  night , she was so  and and and and that  it that that that that  that that that that the  night that that she the  one for her and and  and and that that that  that that that that the  night that that she the  one in the wood which  that that that that that  she that that she that  that that that that the  night , she and the  one and and and that  that that that that that  that that that that that  that that that that the  night that that he that  that that that that that  that that the night that  that she she that that  that that that that that  that that that that that  that that that the one  in the tree which the  king , and the little  ring and and and that  and that that that the  good , and that the  king and and and that  he that that that the  night that that she the  beautiful and which which that  that that that that that  that that that that that  that that that the one  for the king and the  the ring which that the  king , and that that  that she that it the  night , and she the  one and and and that  that that that that the  one , and that that  that she that it that  she that that that the  night that that that that  that that that that the  night , the king was  and and but that that  that that that it that  she that that that the  night that that he the  one for her and and  and and and and that  she that that that the  that that that she that  that that that that the  night , she was so  and and and and that  that that that that the  good , and she the  one which and that that  he that that that the  that that that he that  that that that that the  night , she was so  and and and and that  that that that that the  good , she and that  she that that that that  she that it that the  night , and that the  night , and she that  it that that that that  it that that that the  night that that he that  that that that that the  night , she and the  and and and and that  it that that that that  it that that she the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 41)\n",
      "(41).(66).(2).(12).(49).(0).(79).(89).(73).(21).\n",
      "Average loss at step 42: 3.411028\n",
      "\tPerplexity at step 42: 30.296375\n",
      "\n",
      "Valid Perplexity: 80.67\n",
      "\n",
      "Generated Text after epoch 41 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " UNK . he and he and  that the father , who  , and that that the  well , and he the  one and and and the  one and and that that  all the good i have  it for you , and  you have the good man  . he , and the  good , and he he  he and that that the  one , and that that  that that that that that  that that you you you  you as as as i  do , i can not  so so you i have  that and i have no  little and and and and  that that that it the  well . he and the  good , and he and  and and that that the  father , and he and  and that that that the  one , and that that  for it , and the  father and and and he  , and that that that  that the good you can  come and and and he  in the other time ,  said the father , and  , he , and that  the father , and the  father and and and and  all that that that that  the good i have the  father you and and so  good you i can no  and and but and as  as as as i i  as as i have ,  you have the good and  for you i have the  good blow , and the  mother and and he he  in the ground , and  that the good man was  and but but so so  and he he he that  what he that that the  father , he and and  in the other other time  , and it he the  so and and that that  that the good , he  and and and that that  that that the good ,  said the father which he  , and he and the  so and and he he  and and that that that  that the good , said  it and the father who  in the ground , he  , he and and and  in the other time he  came to the other other  time , and the father  and and he and and  in the other time that  that that it as the  one , and he the  so and and he he  and and and that that  that the good , he  and and and that that  that that the good ,  said the father which he  , and he and the  so and and he he  and and that that that  that the good , said  it and the father who  in the ground , he  , he and and and  in the other time he  , and he and the  so and and he and  what that that that the  good , and he he  in the other side and  it that the father ,  he and and and that  that the good man ,  he and and and and  that that that that that  that the good i have   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 42)\n",
      "(37).(88).(12).(42).(9).(38).(50).(67).(35).(43).\n",
      "Average loss at step 43: 3.509016\n",
      "\tPerplexity at step 43: 33.415378\n",
      "\n",
      "Valid Perplexity: 79.83\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 42 ... \n",
      "======================== New text Segment ==========================\n",
      " was a little man which the  man and and and the  one and and that the  father and and and that  he that that the little  king and that that the  night that the little man  , and the little man  , and he and the  one and and and the  little other other which the  man and and and the  father and that that the  night that the little man  and the little man in  his other house and and  and the little man and  and that that that that  the little man was so  so that the little man  said , that is not  my father , and is  not my father and have  that my father , and  he the little man was  in the earth and and  and the little man and  all that that that a  good man and the little  little daughter , and that  that the little man was  in the room that that  the little queen came so  that the little man was  all and and that the  one which the little man  , and he and the  two and and and the  one and and the other  , and he and that  he that that the little  king and that the little  king and the little daughter  , and he and the  other that that that the  night that the little man  and the little man in  his other men and the  son and and and the  one and and the other  , and that that the  night that that he the  one in the earth the  man and and that that  the little queen was so  the man and and the  little man and the other  which that that the little  little man came and and  it that that that a  good man which the little  little man and that that  a little queen was so  and and the little man  , and and that that  the little king was so  but that he he the  two and and and the  one and and the other  , and he and and  the other other and other  which that that that the  three man and and the  father and and and the  one and the other and  who that that that that  my little man , and  the little man and the  son and and and that  he that that the little  king came and and the  last and and the other  , and he and that  that that that that that  he that my father ,  and the little man was  and and that that that  that the little man and  his little daughter and the  little man and and and  and that that the little  queen came and and the  little man and and that  he that a little man  , and he and so  and and that that that  it so that the little  queen came and and the  three man and and the  one and and the other   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 43)\n",
      "(53).(84).(79).(8).(91).(20).(14).(26).(50).(64).\n",
      "Average loss at step 44: 3.399042\n",
      "\tPerplexity at step 44: 29.935422\n",
      "\n",
      "Valid Perplexity: 80.37\n",
      "\n",
      "Generated Text after epoch 43 ... \n",
      "======================== New text Segment ==========================\n",
      " hand . the king and the  little ring and and the  other which that that that  she that the little tailor  , and she and so  so that the little world  tailor . the king was  and and that that that  that that the night was  in the night which the  king which the little tailor  said , i will not  not so one and the  little soup which that you  for you , i have  you you you for you  , you i have so  one for you for the  king , and she the  night , and that the  night , and he the  little other and and it  that that that the night  that the little tailor was  for the king , and  she and so that the  little tailor which that that  , she that that that  she that the little night  , which she that a  night , and she the  little maiden and and that  she that that that that  that the night was so  good the night she was  but so that that he  he that that that the  beautiful man which the little  man and and and and  and that that that that  he that the little tailor  would not take you and  that you and have a  good work . the king  said , i have the  good blow , and the  good boar will not not  not so one for the  king , and that it  for you , have you  for you , and i  go home , and the  good bird was and and  it that that she that  a little tailor , which  that that that the night  , the little tailor was  on the night which the  good man and and the  father and and and the  that , and that that  it , and she no  one to not take the  three which and and the  be which that that that  you i i you you  so so as you have  you for you , i  do you have you so  long and i have the  little work . the little  tailor says and she the  night , and the little  maiden which that that it  in the world which the  beautiful man which the little  man and and that that  that that that the night  that that the night was  to see it . the  man was and and the  one which that that it  and not so beautiful ,  but the little tailor and  which that that that that  she that it for you  . and the night which  he that it and the  little ring and and that  it , she and so  that that she that it  for her , said and  it and not a little  bird and that that that  that i you you you  i you i i no  one for you , i  do you have you so  as you are . the  first tailor , and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 44)\n",
      "(67).(79).(32).(1).(11).(14).(40).(29).(26).(16).\n",
      "Average loss at step 45: 3.396401\n",
      "\tPerplexity at step 45: 29.856446\n",
      "\n",
      "Valid Perplexity: 78.00\n",
      "\n",
      "Generated Text after epoch 44 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " and that that that she that  the little tailor which the  little daughter which which the  little daughter which that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  there that the night she  was so that that she  that it that that that  that that that that the  little daughter which that it  and not that that that  he that that the little  beast which that that the  good daughter , and the  night and that that that  he that the little tailor  , and he that that  that that that that the  good daughter , and the  father and and and that  and that that that the  that that that that that  it that that that that  he that the little tailor  , and she that that  that that that that that  he that the little daughter  , which she that that  he that that that that  it that that that the  little daughter , and the  one and which that that  he that that that the  little daughter which that that  she that that that that  it that that that the  little daughter , and the  child which that that that  he that the little daughter  , which she that that  he that that the little  woman which that that that  that that that that that  there that that that a  good daughter that that she  that that that that that  there that that that that  that that that that that  there that that that the  night which the little daughter  , and that she that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  there that the night she  was so that and and  it that that that that  that that that that the  that that that that that  she that the little night  that , he and the  little tailor and that that  that he that the little  man which which that the  beautiful daughter , and the  other which that that that  that that that that the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 45)\n",
      "(76).(90).(66).(92).(61).(38).(85).(50).(82).(96).\n",
      "Average loss at step 46: 3.363942\n",
      "\tPerplexity at step 46: 28.902904\n",
      "\n",
      "Valid Perplexity: 76.93\n",
      "\n",
      "Generated Text after epoch 45 ... \n",
      "======================== New text Segment ==========================\n",
      " he she the son and and  it that that that the  great night which she that  it in the tree which  that a great castle which  which that a great castle  . the king which which  , that that that the  great night which which that  all a great castle which  which that a great castle  . the king which the  little son and that the  father which that that the  great night which which that  a great castle which the  king and son and the  son and that that the  great night which she the  that , and that the  father and that that she  she that that she that  that that she that it  that that the night the  little night which was so  beautiful and the king which  that that a great castle  that that that that that  she that that she the  night in the tree which  it was a great castle  . and she that it  and that that she the  that , and she the  king and that the father  , and that she the  that , and he the  king which that that the  beautiful king , and she  and the one which the  son which that that the  great night which was a  very castle which that that  that the night , and  the king , which the  son and and all the  king , and she the  little son which that that  she that that she that  that the night that that  that the night that the  great night , she was  in the great castle which  that that she that that  that the night that that  he she that that she  she that that she that  there that that the night  which that the night ,  and the king which the  father and and and and  that that she that that  it that that the night  , the father which that  that that she that it  as that as she the  one , and she that  that that that the night  , and the king ,  which she that that she  she that that she that  there that that that that  that the night that that  the night , she the  son and that that she  she that that she the  that that that she the  good , and she the  little son and that that  she that that she that  that that the night that  that she she that that  she that it that the  father , and she the  little son which that the  great castle which was a  very castle which that that  it in a great castle  and which that that she  was a great castle and  but which that that that  that the night which the  king and son and the  old king which which that  a great soup and the  old king , and she  , that that that the  night , she was so  white that that the king  which which that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 46)\n",
      "(88).(98).(86).(82).(43).(36).(94).(11).(54).(79).\n",
      "Average loss at step 47: 3.383870\n",
      "\tPerplexity at step 47: 29.484645\n",
      "\n",
      "Valid Perplexity: 87.07\n",
      "\n",
      "Generated Text after epoch 46 ... \n",
      "======================== New text Segment ==========================\n",
      " forest . the child which the  child which that that that  that that it that it  it that that it the  night that that the night  it that it that it  it it it it it  that it that the night  it that it that that  it it it it it  that that the night that  that it that it the  that that the night the  old queen was all the  child and and it the  that which that it that  that it that that it  it that it that the  one that it that that  that it it that the  one that the night which  it that it that that  it it it it the  night it that it that  it that the night the  good palace , she it  in the little palace and  that the little boar was  for delicate little little child  the little child which the  good man and the little  king and all and the  one that the little child  that the little boar was  for gold . the child  which that that that that  that that it that the  one that the night which  it that that it that  that it it it it  there it that it that  it that the night the  good night , the little  child which that that that  that that it that it  it that that it it  there that it that it  it that it that the  one that the night the  little tailor came and the  child and but it that  that that that that the  all that that the night  it that that the night  it that that it that  that it that the night  that it it that the  one which it it the  night that that it that  it that it that the  one which it that that  it that that it the  night that that the night  that it it that it  it that it that the  one that the night the  little tailor came home and  at last that that that  that that that that it  that it that the night  it that that it it  there that it that it  there it that it that  it that it that it  it that that it it  it it it it that  that it it that it  that it that it that  it that it that it  that it that it that  it it that it that  that it it it it  that that it that that  it that that it it  that it that it that  it it it it it  that that it that that  it that it that it  it it it it it  that it that the night  it that it that that  that it it that the  one that the night which  that the night which the  good queen , the child  that it that it the  good be which it it  that that that it it  there it that the night  it that it it that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 47)\n",
      "(54).(46).(85).(2).(82).(57).(90).(60).(67).(52).\n",
      "Average loss at step 48: 3.419942\n",
      "\tPerplexity at step 48: 30.567656\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Valid Perplexity: 85.36\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 47 ... \n",
      "======================== New text Segment ==========================\n",
      " was the king which the king  , and that that the  little king , that that  that that that that a  night , the king was  a king and that the  king , and he the  other and and that that  the king , and the  next morning that that the  night that that he the  night that that he that  it and the king and  that the king and the  father and and all the  king and that that that  he that that that the  night that that that the  one , and he that  all that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  the night you you you  not no one to not  be so one that you  that that that you you  have no one . the  father said , i have  no one . the king  , and that that that  that that that that that  that that that that that  he that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  one there that that that  you that that that you  that that that have no  dead . the king ,  the other other other which  which that them that that  no one and but the  king , and that that  he that that that that  that that that that that  that the night that the  one , and that a  great castle and have the  two king . the king  , and the other other  that that that that the  one , and that the  old king , and the  king , and that that  the king , that i  have no one . the  father and all all all  that that he was so  all that the king and  all that the king which  he that that that that  it that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  you that that that you  not no one to but  but so but that it  in the door which the  first door which he the  king , and all the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 48)\n",
      "(97).(82).(37).(42).(36).(64).(32).(93).(51).(35).\n",
      "Average loss at step 49: 3.351035\n",
      "\tPerplexity at step 49: 28.532252\n",
      "\n",
      "Valid Perplexity: 83.10\n",
      "\n",
      "Generated Text after epoch 48 ... \n",
      "======================== New text Segment ==========================\n",
      " to my father and and the  man and and and the  one that the little hare  , and the little hare  , the father , i  will not not not ''  '' the one and the  father , and the little  cat , and the little  bird was and and the  others , and the little  little man and the father  , and that the little  hare and the little hare  and the little hare and  and the little hare and  the little hare which the  little other and and the  man and and the little  little other , and that  the little boar was so  one the little hare which  that the little man and  that the little hare was  a little bird and the  king and the little hare  , and the little same  that the little bird was  , and the little hare  was all the little and  that the little boar was  in the tree and the  little bird and the little  man and and and the  father and and that the  father , and the little  cat , and the little  bird was and and the  others , and the little  little man and the father  , and the little other  that the little boar was  and and the little bird  and the little hare which  all the little man ,  the father , and the  night , and the little  cat , and he and  the little maiden which the  little daughter and the daughter  and the little hare which  that the little man which  were all all so and  the little bird and the  little hare which the little  other and and the little  little hare which the little  little man , and the  mother and and the little  little other that the little  bird was and and all  once and the little little  man and the little hare  and the little hare which  all the little man was  once and and the little  bird and the little hare  and the little hare which  all the little daughter which  all all and the little  man and the little hare  , and that the little  bird was and and all  one and the little bride  that it and not the  king , and the little  bird was and and once  the little hare which the  daughter , who , the  father , and the little  other , and the little  cat , and the little  cat , and the little  cat , and he and  his other and and the  father and and the little  little other and and all  one that my father and  she the little hare and  the little hare and the  father and and the little  little other and and the  father , and the little  hare and the little hare  and the little hare which  all the little daughter which  all all and the little  hare and the father and  she the little other and  and the little hare which   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 49)\n",
      "(98).(64).(11).(69).(17).(14).(24).(19).(87).(65).\n",
      "Average loss at step 50: 3.494426\n",
      "\tPerplexity at step 50: 32.931391\n",
      "\n",
      "Valid Perplexity: 76.84\n",
      "\n",
      "Generated Text after epoch 49 ... \n",
      "======================== New text Segment ==========================\n",
      " he said , i have no  other , and i do  it so and and you  do you and and the  little door which is the  door . the tailor said  , that that you have  to do it . and  as that as the little  boar which that the great  little door which that it  in the door which the  little daughter which that a  one which have the great  and and it so and  on it , and he  and all and and which  which that it all a  other and and it the  little castle and the little  castle which the little daughter  which he that that the  father which all and and  one and which which that  all and and great great  stone which that that he  that it and all a  little girl which that that  that she that it was  not the little girl which  that the little man and  the little door which the  father which the little daughter  , and that that he  that it was the whole  castle and and the great  stone which that that that  she that that she the  castle , and that that  that the king was so  that that the king was  the little and and the  other and the little daughter  that that that he the  king , and all one  one which that it so  so and he and the  little maiden which that all  which that it and the  end and which that a  good one and which which  that it in which i  not no one for the  mother and and it a  way , and i have  the little and and one  little shoes and which your  little life i have the  king and and the great  door which that it is  the young young bride which  if i do you not  to do it . and  he and all and which  it that that he the  little daughter and which the  little other which he the  little daughter which that it  in the door and the  young king and the little  daughter which the little daughter  all and and which that  that the little tailor was  in the wall and the  young king and the little  castle and the little daughter  all and and and and  it and all and which  which that it that that  that the little tailor was  and and all which the  little daughter which that it  in the door , the  king and what that it  that the little tailor was  and but all all which  that the little daughter and  the door which all all  which that the little girl  should not not not not  not so so that that  the king and the little  king and and all the  other and which that it  that the king which the  little daughter which the little  son was and and the  king and the little door  and which that the great  tree and the little castle   \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "====================================================================\n",
      "\n",
      "Training (Step: 50)\n",
      "(55).(2).(94).(11).(27).(83).(87).(62).(29).(1).\n",
      "Average loss at step 51: 3.337360\n",
      "\tPerplexity at step 51: 28.144723\n",
      "\n",
      "Valid Perplexity: 73.46\n",
      "\n",
      "Generated Text after epoch 50 ... \n",
      "======================== New text Segment ==========================\n",
      " the king , and and she  all that that that that  it that not that that  it that that that that  you that you too in  the door . the mother  said the little child ,  she and that that she  and that that the little  woman was all a little  or which which that no  little house that that i  i not too in my  bed and do you and  look in the world and  not my little fellow ,  i will not not my  little child , and the  little maiden and the little  maiden and the little door  and the little child and  the little daughter and the  house , and she and  that and that that the  old woman , that she  and that that she the  little maiden and all the  little which which that the  little maiden and which the  wife and the little little  man and the child and  in the earth and a  three and and which that  that the little daughter and  the little child and the  daughter and the little child  , and and all her  and and and and and  and that that she that  a little daughter which the  old daughter , and the  little woman and all the  death that the sun was  all a little and which  he that that that the  old woman , and the  one and the little maiden  and the little door and  that the little woman was  all the little which that  the sun came home and  all the little door and  that the little man was  a little daughter which that  you will not not not  so but for you and  look for my little child  that you have not no  and work . i will  not not my little child  . the sun said ,  that i will not not  it in my door and  do my old child and  you will not not my  old child , i will  just not not you so  that that you i i  too in my door .  you will not not not  no so but for you  that you have not so  one for you . i  will not not you and  and my little child and  that you will not not  so so one and the  door . she and the  mother and and and the  one and which that the  little maiden and that the  mother and the little maiden  said that the little sun  came and the little maiden  said the little child ,  you will not not my  one for my mother and  i not not no so  so that it that the  old man , and and  that that the little man  and her little daughter and  that that the little woman  , the child and the  father and and not not  that that that i not  just that you that not  that you , i do  no one . you will  just not not you my   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 51)\n",
      "(24).(63).(70).(91).(73).(23).(81).(17).(15).(16).\n",
      "Average loss at step 52: 3.484443\n",
      "\tPerplexity at step 52: 32.604267\n",
      "\n",
      "Valid Perplexity: 77.15\n",
      "\n",
      "Generated Text after epoch 51 ... \n",
      "======================== New text Segment ==========================\n",
      " , she and not leave ,  she and that that the  little girl which she the  two other and and and  and that that she that  it and not that the  king , and she and  that that that that that  that that that that that  it that it and the  little girl which that the  king which the son and  the son and that the  one and the other and  so that that that she  and that that the little  girl which the son and  which that that that that  that that she that that  that the son which the  son and that that that  he that that and the  two and which that the  little maiden which which the  one and the other and  and and that that she  that the little daughter ,  she and that that the  little girl which the son  , and that that that  it that that that she  and that that she that  it that that that she  and that that she that  that that that that that  it that that that she  and that that she that  it that that and the  king which that it and  the little door which the  beautiful daughter which that that  and not that that the  king which the son and  the little maiden and and  so that that she that  it that that that she  which that that that she  and that that the little  man which the son and  that that that that that  that that that that that  it that it and not  that that that that that  it that that that she  and that that she that  it that that and the  king which that it and  not that that that that  that the little girl was  all all so and that  that that that that that  it that that that she  and that that she that  it that that and the  king which that it and  the little door which the  son which that that the  little girl which she that  , and that that she  that that that the little  maiden which that that the  little daughter which which the  little daughter which which that  no one and not that  . the king and the  father and and that that  that that that that that  that that that that she  and that that she the  little daughter which that the  king which the son and  but and and that the  other and which which that  all that that she the  little daughter which that the  one and the son and  that that that that that  that that that that she  which that that that she  all that that that that  that the little girl came  to the little door and  which the son and and  so that that she that  it that that and the  little maiden that that the  little girl which that the  father which the son and  not that that that that  that that that that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 52)\n",
      "(37).(48).(46).(7).(6).(2).(60).(31).(84).(18).\n",
      "Average loss at step 53: 3.504660\n",
      "\tPerplexity at step 53: 33.270133\n",
      "\n",
      "Valid Perplexity: 75.91\n",
      "\n",
      "Generated Text after epoch 52 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " them . and she and the  maiden and and and and  one and the other and  which that that that the  other which the little maiden  which the little maiden was  home and and the maiden  , and that that the  other which the little maiden  which the little maiden came  , and and and that  he that that that the  other which that that the  night the little daughter which  that that that the little  king came to meet the  door and the other which  which that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  she that that the little  girl came and and and  it that that that that  that that that that the  little maiden came home and  and and the maiden and  and and and that that  that that that that that  she that the little maiden  , and the little maiden  and and the maiden and  which that that that the  little maiden came home and  and and the maiden and  and the other and and  that that that that the  little man the little daughter  and the little maiden which  he that that the little  girl came and and and  and that that that the  one which the little maiden  which the little daughter and  once and and and and  that that that that that  that that that that that  that that that that that  that that that that that  that that that that that  she that that the little  girl came and and and  that that that the little  girl came and and and  that that that the little  girl came and and and  and that that that that  that that that that that  she that that the little  maiden came and little maiden  , and that that the  other which that a little  ring which the little maiden  and the little maiden and  the little maiden and the  little maiden and the little  maiden and the little maiden  and and the other and  who that the other which  that the little daughter ,  which the little maiden which  all a little maiden and  the little maiden and that  that that the little man  the little daughter which the  other which the little maiden  which that the little maiden  , and the little maiden  came and and and the  little maiden and the little  maiden and and and the  one and the other and  which that that that the  good , and the little  other and that that that  the little maiden the little  castle which the other which  that the little maiden which  that the little maiden which  a little maiden and and  once and that that that  that that the little man  which the little maiden which  a little maiden and the  little maiden and that that  that the little man was  the little maiden which the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 53)\n",
      "(56).(82).(76).(15).(95).(98).(51).(86).(48).(93).\n",
      "Average loss at step 54: 3.271723\n",
      "\tPerplexity at step 54: 26.356716\n",
      "\n",
      "Valid Perplexity: 79.88\n",
      "\n",
      "Generated Text after epoch 53 ... \n",
      "======================== New text Segment ==========================\n",
      " down that the little tailor was  once and so so one  as the little hare the  little hare which that that  he all all that the  king which all one all  all all all all the  good one which all so  beautiful that the little hare  which all one all one  one and that that he  he all all that the  little maiden which all so  beautiful that the little hare  which all one all a  beautiful little deal which the  good man and the one  , and the little same  time that they all so  so one that they the  beautiful maiden which all so  and that the little maiden  came and the little hare  and and one which the  little others that that the  one which not so so  one that the little maiden  came and and all all  one so so that the  little hare which the one  which all all so so  that the little hare which  that the little hare the  little hare which the one  for one and the two  which that that they the  little maiden and the two  man and that that the  little others are not so  beautiful that the father is  all that that they is  not one as my father  in the same grave and  that it is not so  but that they is not  one as the little same  little time them . the  little hare which all all  all my good that they  he all all that the  little maiden which all all  one all that the one  , and the little maiden  whom the two others be  the little maiden which the  two maiden and and all  that that the little maiden  which the one which all  all so so that the  king said , the little  same time i do not  but that the little hare  do not do that the  little hare are all a  good one which that the  do do . and as  this as a great deal  for them and all all  so so the little maiden  said , the little hare  i not not so one  that you is not so  but that i i not  do that you is not  one as that as you  do to be . and  and one one and all  all so one that the  little hare which they the  little maiden and the other  , and that that they  he all so so that  that they all one and  that that the little others  do not do that the  little hare which all all  all a one and so  that the little hare which  you not not no one  , and that that he  are all as that as  a great deal and money  and my little bride and  so one which the little  little hare which all one  that the little maiden which  he all all all all  all all all the one  that it as the little  tailor which all all all   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 54)\n",
      "(84).(8).(81).(63).(52).(31).(19).(70).(26).(33).\n",
      "Average loss at step 55: 3.353361\n",
      "\tPerplexity at step 55: 28.598695\n",
      "\n",
      "Valid Perplexity: 76.57\n",
      "\n",
      "Generated Text after epoch 54 ... \n",
      "======================== New text Segment ==========================\n",
      " anger , the little tailor which  which he that that the  little man which he that  in the house which the  little man which the little  tailor which he that that  the little tailor which the  little man which that the  father and the one and  that the little tailor which  that the little tailor which  he that that the little  hare which the little tailor  came and and that that  that that he he that  that the little tailor which  which the other which the  little maiden which that that  that he that that he  he that that he the  other and and that that  that he that the little  hare which the little tailor  came and and that the  little tailor which which the  little man which he the  little man which he the  king and the other which  the little tailor which the  king came and and that  that that that he the  other , and that that  that he that that he  he that that he the  other and and he that  he that that he the  other and and he that  that that he that that  that he he that the  little tailor which which the  one and the other which  which he that that he  came and and that that  that he that that he  he that that he the  other and and he that  he that that he the  other and and he that  that that he that that  that he he that the  little tailor which which the  one and the other which  which he that that he  came and and that that  that he that that he  he that that he the  other and and that that  that he that the little  hare which the little tailor  came and and that the  little tailor which which the  little man which he the  father and and the other  , and that that the  little man which that the  little man which that the  little man which that that  that that that he that  that he he that the  little man which he the  father and that that that  that he that that the  little tailor which he the  other which the little tailor  and he that that the  little tailor which which the  other which that all that  his little tailor which the  little man which he the  father and and that that  that the one which the  little man which that a  little man and that he  was so long that he  he that that the little  hare which the little tailor  came and and that that  that that he he that  it in the house the  little son which he the  one and the other and  he that that the little  peasant which he that that  that he that that he  that the little tailor which  which he all that the  little man which it the  good maiden and the little  hare which he that the  father and the little tailor   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 55)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(30).(24).(92).(45).(88).(34).(67).(75).(17).(26).\n",
      "Average loss at step 56: 3.384006\n",
      "\tPerplexity at step 56: 29.488678\n",
      "\n",
      "Valid Perplexity: 76.75\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 55 ... \n",
      "======================== New text Segment ==========================\n",
      " all a great stone and and  it that that that that  that the little man was  all so that that she  which that that that the  little man was all a  wicked and and which it  that that the fox came  out and and that the  little girl and not not  leave . the man said  , i have not so  long as you do ,  and i not not so  that that that i not  no one , and not  not long for you that  that i have not so  well . and she the  other and and that the  little girl came home and  not that , she the  two maiden and and the  father and and that that  that that that that she  he that that he the  son which that that that  was all all and the  young man and that the  little girl and not not  leave . the man said  , i have not so  long as you do ,  and i not not so  so long and i have  that for you . the  father said , i have  not so long as you  are . you have not  for that and have not  so so long and your  well . i have not  all so so that for  you , i have not  all so so that for  me , i have not  all so so that ,  you are not so long  that you have not so  long and have not a  long one and that you  is not so long for  you . i have not  all so so that ,  you are not so long  that you have not so  long for you i do  not not so that the  old man said , i  not not so long for  you i have not so  well and have not the  long one which that my  one . that i not  not so long as the  little door which i not  take you home . the  father said , you are  all so long and your  one . you have not  all so long and your  one , and not not  long as you do ,  for i have not so  well . and i not  no one , and not  not long for you .  i have not so so  so a long time the  good thing i have not  that that that i have  not so so that ,  you are not so long  that you have not so  long and have not the  long one and you you  that that you have not  so so long and your  one . you have not  so so that , the  little tailor said that she  all not that that the  little tailor was and but  it and all all that  that he was all all  all and he that the  little man which that the  little man which that that  that i not not that  it as the fox and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 56)\n",
      "(13).(90).(95).(11).(62).(50).(27).(34).(59).(21).\n",
      "Average loss at step 57: 3.348692\n",
      "\tPerplexity at step 57: 28.465470\n",
      "\n",
      "Valid Perplexity: 74.20\n",
      "\n",
      "Generated Text after epoch 56 ... \n",
      "======================== New text Segment ==========================\n",
      " with them . she and the  little man and that he  he that that she the  little maiden and that that  that the king was so  small and that that the  old woman she that it  that the king which the  little man and that and  and that that he was  all her and and all  that he and that and  and that that he the  little man and that he  she that and and the  father and that that he  came and and it that  that the king and that  that the king and all  it that that he the  little maiden and and that  that the king was so  long that the king and  it that the king and  it that that he he  all that that she the  little man and that that  the king which all a  good white which which the  other which which that and  all all all all a  good and which he the  little maiden and it the  good home and and what  she all that that she  came to her well the  king and that the king  came home and and the  father and the one which  that that that the king  came home and and he  at once that that she  she the other and and  that that she was so  long and all that he  came to her great house  . and she and that  that the little girl came  home and said that he  all not just not so  well . and he the  little maiden and that the  little man was and the  little man and she the  father and that that the  little man which she the  father and and that that  that the king was so  small and that that the  old woman she that it  that the king which the  little man and that and  and that that the king  came home and said and  you i not not not  too so long . the  father said , i have  my bride , i have  you to go home and  you and i have the  good fellow and have the  wedding . the king and  he and that that he  he the other and and  the other and which the  little man and the father  and that that he she  and that that he the  white maiden and and the  maiden which that the king  , and that that she  was so long that the  king and and that that  that the king was so  small and that that the  old woman she that it  that the king which the  little man and that and  and that that the king  came and said that the  little woman was so great  which the little woman was  home and and the king  , and she and all  and that that he was  not so long and the  old woman she and and  that that he was so  long that the little girl  and the little maiden and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 57)\n",
      "(50).(40).(94).(71).(91).(7).(21).(96).(95).(49).\n",
      "Average loss at step 58: 3.393563\n",
      "\tPerplexity at step 58: 29.771840\n",
      "\n",
      "Valid Perplexity: 74.00\n",
      "\n",
      "Generated Text after epoch 57 ... \n",
      "======================== New text Segment ==========================\n",
      " brought his little wife and the  three man and his other  who which that that the  queen which that the little  white man which he the  beautiful man and him and  that that the king and  it that the king and  all the little maiden which  he that for a great  tree which the king which  that the little man which  he that that the king  , and he and that  that the king which the  white man and the other  which which that and the  little man which which the  great man which that that  it that it and the  man which that that that  he that that the king  were all all that he  came to the great forest  , and that that that  that the man which the  great man which that that  it that the king and  all the little man which  all the other and which  and the other and the  little man which which that  it that that that he  that the little man which  he that that the king  , and that that the  little man was all all  that the little man was  a little white which which  he that and all the  man and the little man  and the other which which  all the other and the  little maiden which that that  that the king which the  little man which he that  it that the little man  and the king and the  king and that that the  little man was and that  it that the king which  he the other and the  little maiden which that the  little man which he that  for the king and the  other and which that a  good man which the king  , and and that the  little man which he the  little man which that the  little man which that that  all all it that it  for the king which the  three man which he the  that that that the king  he that that that he  it that that the king  were all all and the  little man which he the  other and the other which  it that that the king  were all all that he  came and all all a  three which which that the  man which that that the  beautiful daughter and the little  man which the other which  he that that the king  were all all that the  little man was all the  little which which that that  that the king and the  three man which that a  one and which that all  a great castle which the  three man which that it  and a great castle which  that the king which the  three man which he the  that that the king which  all the other and and  the other and which the  three man which that that  it that it and the  little man which that it  and the king and the  other and the other and  that the little man which  he that that the king  who all that that the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 58)\n",
      "(30).(88).(74).(47).(19).(91).(26).(73).(57).(29).\n",
      "Average loss at step 59: 3.312272\n",
      "\tPerplexity at step 59: 27.447416\n",
      "\n",
      "Valid Perplexity: 76.44\n",
      "\n",
      "Generated Text after epoch 58 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " you will not not soon leave  it in the table and  do you do you so  so as the old woman  he said that the old  old woman was all so  great which that as the  old man said , i  do not know what the  old king , and i  said , i will not  soon soon soon so that  as the old woman was  very very very that the  whole table which he the  old man and and that  as the old woman came  to the end and the  beautiful maiden which that the  beautiful man and that that  he that that that that  that he was so very  the maiden that as the  little woman which that the  beautiful man was so very  one for him the table  which the old man was  and and that that the  beautiful maiden which that a  beautiful man and the maiden  who the beautiful maiden which  he that there and a  little man and the maiden  in the whole table which  he that and and the  beautiful maiden which that that  as the old woman was  once and her a great  very which which which that  as the old woman was  once and all a great  very which which which the  three which which that a  little which the old man  he and that that the  maiden which that that that  it that the old woman  he the beautiful maiden and  it and all that that  the maiden which that the  little maiden which that that  as the old woman was  very very very that and  that it and all so  that that as the old  old woman was all a  little which he the very  man and and that the  little maiden which the beautiful  man which he the other  , and and that that  that that that he the  other and and the maiden  and that that that that  it that that as the  old woman was all home  and and it and the  old maiden and the maiden  which that that that the  beautiful man and the maiden  and that that that that  that he he that the  little maiden which the beautiful  boy and the maiden and  the other and which that  it and all a great  great which which that that  that a great maiden and  that that that the very  man which that that that  all the beautiful man was  and but all all a  great which that that that  all all all the great  that that that that the  beautiful man which that that  there that a great table  , and the maiden which  all so that that the  other maiden which that that  in the ground , and  this the old woman was  once so so that that  he was a very very  which that in a sack  the table which all so  long that the maiden and  all all the great which  he that and and that  that that that that the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 59)\n",
      "(98).(96).(10).(21).(32).(97).(56).(58).(15).(18).\n",
      "Average loss at step 60: 3.264816\n",
      "\tPerplexity at step 60: 26.175301\n",
      "\n",
      "Valid Perplexity: 81.56\n",
      "\n",
      "Generated Text after epoch 59 ... \n",
      "======================== New text Segment ==========================\n",
      " garden for the little table which  all the other and which  all the other and which  it and that the little  little maiden which that it  that the little maiden which  which the little maiden which  which that that the little  tree which the little maiden  came and and the little  little man and the maiden  came and and that that  the little maiden which the  son which that the little  little maiden which that that  a little maiden which the  little maiden which that the  beautiful maiden which the other  , which the other and  one and which that that  she came home and the  little ring which the little  maiden which the other which  all and that that the  other which the other which  which the other which the  little other which the other  which which that and the  beautiful maiden which that the  good maiden and which the  little maiden which the little  man and the other and  that the little maiden which  which that that the little  tree which the little maiden  which the other which which  the other which the other  and which which that that  she came home and the  other and and other which  that the little maiden which  all and the other and  other and which which the  other which the other which  which that and the other  for the other which the  other which which the other  , and and that that  that that the little man  which that that the little  girl came and and the  old maiden and the other  other which that the little  little maiden and which the  little maiden which the other  and which which that that  she came home and the  three maiden which that that  the little maiden which the  other which which the other  and which which that that  and the other and which  all that that the other  who and that and that  and the other which that  it and the other and  which the other which which  all and that that the  other which the other which  all and the other which  the other which which that  that the little maiden was  for the little little which  it and the other and  which that that the little  sisters which the little maiden  came and and that the  man and the other and  which the other which the  little maiden which the other  other which which the other  which which that and the  son and the other which  the other which which that  and the other and which  which that that the little  man came and and each  that the little maiden which  all and that the little  man came and and each  and the other which the  fox came home and the  beautiful maiden and the little  maiden and the little maiden  that that the little maiden  which the beautiful maiden which  that that that the little  man came and and the  other maiden which that the  one which the other which   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 60)\n",
      "(46).(59).(83).(20).(7).(94).(30).(45).(58).(85).\n",
      "Average loss at step 61: 3.091092\n",
      "\tPerplexity at step 61: 22.001085\n",
      "\n",
      "Valid Perplexity: 79.00\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 60 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " she all all all all all  all all all the little  man which the little which  which the little man which  which the other and all  for all the little man  were once and the little  which the little man and  it and all and the  beautiful man and the maiden  which the other other which  all and and all and  the other which the other  dress which the other which  which the other and that  the other which the other  man and the other and  the other which the other  and which the other which  and all and all all  all all all the little  other which the little man  were once so a great  and which that it the  beautiful man and the little  man and the maiden and  once for the maiden and  and the little man and  the little man and the  old man and the maiden  who and that that the  other and the other which  which all all all the  dead which which the other  which the other which which  all and the other which  and all and the other  and and all and the  man and the other and  all and all all all  all the great which the  maiden which the other and  which the other which the  son and and the other  and which all all the  other which the other which  the other which which the  little other and which the  other which the other which  that all and the little  man and the maiden and  once and the little man  and the little maiden which  all all all the great  and which which the other  and which the other which  which the other which the  other which which the other  of that the other and  it and all and all  all the little which the  king and the little maiden  to not hold it for  her one of them a  great glass which the little  wife and the maiden and  all all a little which  all all all a great  great which which that it  for all all which the  old man and the little  maiden and all which the  dead man which the other  which the other which which  it and the other which  which all all the great  man which the other which  all and the other and  other other which the other  other and which the other  other which the other which  the other which which the  little other and which the  son and and the other  and which all all the  which which the other and  all and all and the  dead which which the other  which the other other which  which all the other and  which the other and which  the other which the other  and which the other which  that the other which the  son and and the other  which the other which the  son and and that the  one and all and the  king which all a little  little which the little man   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 61)\n",
      "(23).(62).(95).(14).(53).(25).(94).(29).(71).(76).\n",
      "Average loss at step 62: 3.269659\n",
      "\tPerplexity at step 62: 26.302369\n",
      "\n",
      "Valid Perplexity: 84.94\n",
      "\n",
      "Generated Text after epoch 61 ... \n",
      "======================== New text Segment ==========================\n",
      " sons to eat , you will  do you . you are  nothing with you . and  when the old man was  and and all that that  it all all all the  one and that all the  little daughter and the son  , and all all all  a dead and which all  which all all all the  all which the old maiden  which which all all the  that that the old woman  which all all all all  that that he were the  bird and all the little  man and and all the  other which all all all  that and all all the  children which the old woman  which that all and all  all all all all the  children , and and that  at the other , and  no one , and all  all all all all and  all all all and and  all all all and the  children and and all all  all that and the other  with the maiden , and  all all all all and  all all all and all  all and all that and  that the old man was  all all all all and  at the other which the  maiden came and and the  old maiden and the other  and and and that that  that the old man came  home and cut the water  all which all all all  that and and that that  all were all all and  and all all all the  children and all all the  children and and that that  all were all all and  and all all all the  children and all all the  three and and that that  that the little maiden were  all and dead which the  old maiden and that the  three sisters came and and  that that the old woman  which all all all all  all all all all and  he all all that the  young horse was all and  so that the old man  came home and cut all  great a little meat which  all a little bird which  which the other and the  son and and that that  that the little maiden which  all all all all and  all all and and all  that and and the other  and and all that that  that the old woman were  very dead which that all  and not that and the  king and not not that  that the old horse ,  where not not that you  that that you are not  as long as you are  with you . and the  old woman said , you  will not be so good  for you , and not  that you are not so  small for this that you  shall not hold that the  riddle and you be a  very bird , and not  come . you are not  so long and that ,  i not not not hold  for this which you are  not so long and be  all or be . but  she and all that the  other and and all the  children and all and the  three other and the other   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 62)\n",
      "(72).(66).(93).(3).(34).(61).(30).(54).(58).(71).\n",
      "Average loss at step 63: 3.203924\n",
      "\tPerplexity at step 63: 24.628983\n",
      "\n",
      "Valid Perplexity: 82.53\n",
      "\n",
      "Generated Text after epoch 62 ... \n",
      "======================== New text Segment ==========================\n",
      " child , and the old man  which the old man and  all all all the dead  and which which which the  other and and the other  , and that the old  maiden were all all and  and all all all all  their all and and all  and and all and that  his other and and the  man and that the old  man and the other sisters  were all dead and all  dead which all the little  bird and the bird which  it and all all and  a dead which that the  little bird which the old  others which the old man  were all all and the  home and and the other  man and them all and  all all all all all  a all which the old  man and and the other  man and each other and  and the man and the  maiden and the other other  of four other fingers which  a little bird which the  dead man and and and  all and all and the  little other which the other  other other and which which  which the other and and  all and all and all  all all all all all  a all which the old  man and all the other  and and each other and  that the old man were  home and and the other  others and drink and if  the old man was all  once so all the little  man and the other sisters  , and the old man  were all all all all  for the other which the  wolf and all all the  dead and which which the  old other and and all  that the old man and  that the old man was  all and the other and  that the old man were  so dead which the old  old sisters which the other  of other which the old  old woman was all and  all all and all all  all all all all all  for the other and all  it and all all and  that the old man was  not dead and and the  old man and the old  maiden to each other and  that the old man was  all all all and the  dead which which the other  other and which which the  other which the other which  all and all all and  his other and and that  he the other and and  that the old man was  all all all and the  which which the other other  other other and which all  all the other which the  old man and the other  man and each other and  dead and that the little  old man were all all  all the little and which  his other which all and  all all all all all  all all all and the  little other which the old  old woman which the other  which which the other which  all and all all and  the other and other other  for the other which the  little old woman was all  which the old man and  and the other sisters and  that that the old man   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 63)\n",
      "(37).(52).(55).(46).(39).(13).(88).(59).(22).(9).\n",
      "Average loss at step 64: 3.350967\n",
      "\tPerplexity at step 64: 28.530311\n",
      "\n",
      "Valid Perplexity: 76.33\n",
      "\n",
      "Generated Text after epoch 63 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " a tree . the king and  that the king was and  that the king which the  children which that that the  king was so little one  that the old woman was  not and cut the little  tree which the king which  it and and the king  was so little one and  that that the king which  it and and that the  old woman said , that  i do what my dear  wife , the king ,  and the king which the  queen and the other and  so that that the king  was so great which the  king and that the king  which the king which the  daughter and and that that  that the king was and  but the king and the  king and the other and  the other and that the  old man was and and  it and that the king  was so great which she  and the king and that  it was not a little  and and that the king  came home and the king  and the old maiden was  and and that the king  was so great the children  which that it was a  children which was the children  which the king which the  queen and the children which  all so that the king  were so great which the  old man which which the  other which the other which  that that the king was  all and and that the  old man was and and  that the king which the  old man and the children  came home and the children  which that that the king  had a little daughter and  that the king was so  wicked that the king which  that the king came and  cut the glass which the  king and that the king  , and that the king  was so great which she  and that the king which  it and and that the  old woman said the king  , and that that the  old king said , what  i do that the king  is so long and the  old man said , what  do you go home and  fetch my wife and the  king , and the king  which the king which the  daughter and and that the  old man was and and  that the king which the  old man which that that  that the old man came  for her little wife which  a little men which the  king and that that the  king which the king which  that the king was so  long that the king which  that the king which the  old woman and that that  that the old man was  and and that that the  king was so great which  that the old man was  and and the king and  that the king and one  of them and the king  , and that the king  was so great the children  which that that the king  was so great that the  king was so little that  that the king which the  old man and the children  that the king was so  little long long one and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 64)\n",
      "(0).(30).(72).(65).(89).(27).(32).(39).(63).(56).\n",
      "Average loss at step 65: 3.095522\n",
      "\tPerplexity at step 65: 22.098775\n",
      "\n",
      "Valid Perplexity: 83.04\n",
      "\n",
      "Generated Text after epoch 64 ... \n",
      "======================== New text Segment ==========================\n",
      " loved her and and and the  cook and that that the  fox which the other and  other that that the other  which the other which the  one which the other which  that the other and the  old man which the other  in the forest which the  little witch which that that  that the old man was  all and all all and  all which that a great  deal which which which the  one in the forest she  in the midst which the  fox which the other and  which the other which that  that that lay on the  water and which the other  the other and the other  , and that that the  fox which the other which  which the other which that  that that she that a  little dog which which the  other which that that and  that the old man was  still and and and one  which the other which the  other which that that and  have the other her and  that the old man was  still and the other and  which that that that the  father was home that that  he was so dead one  which it so so the  children that he was not  so long that he were  once so so the little  maiden and one one and  one which that a little  stone and the little dog  home home and said ,  do you do you do  you do you do you  you do you do you  good that i have you  do not leave you that  i do you do you  do you do you do  my good wife and you  will not do you one  which you have my dear  child , and not be  an old man which the  one have not a little  fellow , you do you  good that i have you  will not do you good  that i do not leave  me , i do you  want to go home and  you and go home and  go home and be my  children and do you do  my good wife , i  i do you know you  will not leave you and  said , i do not  do you that i do  my dear bride i do  you father , i do  not leave you that the  godfather is so dead and  that that the old witch  have to go home and  sleep , he have not  you do my good one  in the forest , i  do not leave you that  and do you do you  one in the world the  fox have you do you  have you do not leave  it and the godfather ,  if you have not you  do not leave me that  that i do not leave  it . the man was  home and and the other  which that it and not  not that that the old  witch was so dead which  it in the midst which  a little dog which the  other which the other which  which the other which which  the other which the other   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 65)\n",
      "(33).(45).(59).(94).(88).(96).(44).(17).(36).(80).\n",
      "Average loss at step 66: 3.165782\n",
      "\tPerplexity at step 66: 23.707277\n",
      "\n",
      "Valid Perplexity: 80.55\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 65 ... \n",
      "======================== New text Segment ==========================\n",
      " said the wolf , `` i  not not not no one  . you have no other  one for him and have  no one . you do  not so one that you  have no other one for  him and go home and  fetch you and go home  for you . and the  poor horse and the little  dog which the wolf which  all the other one was  and and the other maiden  for him and the last  the maiden which that that  one one for them and  all the gold that the  fox was not all so  one and the other one  were a other and which  he the other and and  all the other one and  a great gold which the  beautiful maiden and the other  of them and and the  one and the other one  that the old woman was  not and the other one  for her one and the  glass which the gold which  it so so that the  maiden and the other maiden  which the other one which  which the other one and  the other one and the  third time the wolf which  he the other which the  maiden which the other one  for him and the last  one of him and the  poor horse which that that  the wolf was all so  so that the wolf was  dead so long and that  she was not so so  a long one and the  poor horse which the three  other horse and the one  and the other and that  that the poor maiden was  for the other one and  church the godfather , and  she the other one and  one and them and the  children and the other one  for her little horse which  he the other and which  it the other one and  them and that that the  poor maiden was so beautiful  one which the wolf which  all the other one was  for the other one and  see that the wolf was  dead so long which the  poor dog and the other  horse and that that the  poor horse and and one  for them so long and  he the other which the  other which the other which  it so so that the  little dog which the other  which the other which the  old maiden which that that  the beautiful maiden and the  other one , and the  other one , and the  wolf and the other one  , and the other one  , and that that that  that the other which the  other one which the three  one was so other that  he the other which that  it and not the third  , the wolf and the  old maiden was home and  how the maiden came home  so long and the other  other that that the other  one was so dead one  he the other which the  wolf which the other which  it so so that the  maiden and the wolf which  it and that that that  she the other and and  the other one and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 66)\n",
      "(74).(51).(15).(39).(55).(45).(26).(75).(31).(90).\n",
      "Average loss at step 67: 3.255051\n",
      "\tPerplexity at step 67: 25.920949\n",
      "\n",
      "Valid Perplexity: 80.96\n",
      "\n",
      "Generated Text after epoch 66 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " an fox and the last man  and each other which he  have not so that the  whole castle which which the  king and son and the  three maiden , and the  three man and the other  and which the other which  the other which the other  man and son and the  other and which the other  , and that the maiden  which the other maiden and  maiden which the other maiden  which the other son and  who that the maiden which  that a great castle and  rest for joy and and  once so that the maiden  which the other maiden which  he the other and and  once and that that the  little maiden which that a  great castle which that a  long time which it the  little which the maiden came  for the maiden which the  king which the other which  it so so that the  little maiden which the other  other which which the other  which which the other which  the other which which the  son and the other which  which the other of other  of wood which which the  little maiden which that a  great castle which that it  in great glass which the  beautiful maiden , and that  all a great castle which  it in a great castle  , and the maiden came  so long and the man  were not so that that  he was not so that  he was not so that  that the whole castle which  he the other in which  it the other which the  maiden and other which and  that which the other which  the other which which the  king and other which the  son which that that that  he the other in the  water which the other man  and son and the other  and which the other which  which the other which which  the other of other which  the other which the other  , and that the maiden  , and not not that  that and said that the  king was so long and  that the maiden which the  king and son and the  dead man , and the  little maiden which the other  son which the other of  which which the other maiden  which the other which which  the other of maiden which  which the other man which  he the other and and  once and that that the  little maiden which the other  maiden which the other maiden  and other son and each  other that the fox was  once so a long long  which which it in the  great castle , and the  one and each other and  said , i have a  great castle which you have  the king's man and be  long as long as you  do . '' the man  came and and the maiden  which that the other man  with the water which the  other which the other maiden  in the wood which the  little maiden which that all  in great glass which the  little maiden which that the  one which the other of  which the other maiden and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 67)\n",
      "(55).(1).(30).(40).(75).(35).(29).(32).(7).(24).\n",
      "Average loss at step 68: 3.213428\n",
      "\tPerplexity at step 68: 24.864167\n",
      "\n",
      "Valid Perplexity: 81.05\n",
      "\n",
      "Generated Text after epoch 67 ... \n",
      "======================== New text Segment ==========================\n",
      " of him , and the old  bird said , what my  child , and not not  only that the godfather and  do my child and do  the dear wife and be  once so so as the  bridegroom , which all for  him . and as he  he the other and the  two other of him in  his hand which he so  and the one and her  and which the old man  he the other and which  it in the whole tree  . he all and the  little woman which which that  that that he all so  long and all all so  that and in the evening  it he he and the  that that the godfather was  once so so and the  one which the other of  the other of it the  beautiful daughter , which all  in the air which the  whole night which he the  other of the door ,  but the old woman was  all and all all which  which the other and which  which all in the bed  that the old man was  very very long which which  which all and all his  all and and all which  it so so that he  would not know what the  little woman had to be  the other of them and  and one one and that  that he all and the  long one which all so  so that it and so  long and all and all  all all all which the  dead which which the other  , and that the king  would not know what the  godfather , and there all  and all all all all  which all and all all  once so all so the  man and and and this  and the other of him  that the godfather were so  beautiful that the godfather was  all so long and the  king and his other sisters  which which all all which  it all so long which  the godfather which all so  long as the old woman  he the other and which  which all in the great  which which the other of  his other daughter which he  that and he all in  a great which which the  one , and the other  of him in the air  . the king and the  king and the other of  him in the air and  which that the king and  the other of him in  a great tree which he  all and and the one  that the old man were  all all all which the  godfather which all and all  all all all all which  all and all all all  all all all which all  for her one which the  king which the godfather were  very very very long and  that that the godfather were  very very very long which  which all and all all  and all all all which  all and all all once  all and all all the  beautiful which which that the  other which the other daughter  , and the other daughter  in the ground and once  and and that the old   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 68)\n",
      "(28).(49).(30).(45).(77).(82).(6).(1).(84).(25).\n",
      "Average loss at step 69: 3.236428\n",
      "\tPerplexity at step 69: 25.442668\n",
      "\n",
      "Valid Perplexity: 79.73\n",
      "\n",
      "Generated Text after epoch 68 ... \n",
      "======================== New text Segment ==========================\n",
      " and said that i not not  so long and i not  to be so as it  for that that i have  it , and i not  have not long so that  you are not so long  , and i not not  not not long long .  she said that the king  are not long and not  long and i not not  to be so as it  for my child . she  will not not what it  . the king and the  queen said she and the  old man is dead and  all long long and and  and the one and the  daughter and the door which  the godfather which all for  this the man said ,  i not not not not  long long . she said  and is not long long  or i i not not  be so long and the  old man and the king  who not not long for  this . she said ,  i not not not not  so long for it .  she said that the godfather  is so long one and  he not not long for  it and said , what  i not be so long  that you have not for  the old man , i  not not not not not  not not long long .  then she not not not  no one . and said  the king and what not  that and i not not  so long and said ,  i not not not not  long long . and said  the king and what not  long for her long and  not not not long long  for the king and not  leave home and said that  if she have not long  long and is so long  which you have not to  go home and be my  men and do you not  so long for it and  said and i not not  my daughter and you be  a long time and the  fire for the door and  you not not so long  as i have to be  so that i have not  so long for you and  i not not not long  my long long one and  do not be so long  and that the godfather is  all so long which the  godfather and be long and  a long man which is  long and long one and  said , i not not  just not not my wife  . then she and the  old man she that the  king and the king said  the king and not not  what it for not that  that she is so long  and the king and not  be long for long and  or do . she said  and that that she is  not long long and all  that the king and not  be long as long as  that is a long long  time and i not not  so long for it ,  or i not not not  long long . and said  the queen and the godfather  were not so long and  said the king and not  not not not not that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 69)\n",
      "(3).(34).(33).(14).(66).(8).(2).(80).(21).(19).\n",
      "Average loss at step 70: 3.421432\n",
      "\tPerplexity at step 70: 30.613207\n",
      "\n",
      "Valid Perplexity: 76.21\n",
      "\n",
      "Generated Text after epoch 69 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " the miller which the wolf which  that the little tailor which  which the other which the  boy which the other which  which the other which the  old woman which the other  that the wolf which the  old man came and the  little tailor which the old  woman which the old woman  had not the other one  . the king and the  king and the other man  came home and put the  door which the wolf which  all the great which the  wolf which the other which  the other which which the  old man came and the  little tailor which the other  that he was to keep  the door and the other  that the wolf which the  other which the other which  that the other which the  wolf which the other which  he the other which the  wolf which the other and  the other which the wolf  which the other and the  boy and he the other  that the wolf was so  so that the wolf which  all the great which the  old man which he all  at each other and said  that the king was so  dead that that he was  all so long and the  other which the wolf which  that the king was and  and the last which the  old woman which the other  that the wolf came home  which the wolf which the  old man came and the  little man which that that  the wolf came and that  he the other which he  he the other and the  boy which the other which  which the other which the  wolf which the other and  the other and the other  one and not not what  he the other and the  boy which the other which  that the other which the  wolf which the other which  which the other which he  that the whole which the  third which the wolf which  she the other which the  wolf which the other and  the other which the wolf  which the other which he  that the whole which the  third which the wolf the  other which the wolf which  which the other which the  old man came and the  little tailor which the other  maiden which the other daughter  which the other which the  boy which the other which  he the other which the  king and that the wolf  came and that the wolf  and the other which the  little tailor which the other  which the other which the  old man came home and  said that the wolf which  that the old woman which  that the king's man was  all so dead that the  little tailor which the wolf  came and that the wolf  and the other which that  he the other which the  king which the other which  she the other which the  old man came and the  one and the other and  that the wolf was so  wicked that he was so  very long that he the  old woman which the little  tailor which the other which  he the other which the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 70)\n",
      "(69).(7).(4).(87).(66).(92).(19).(42).(16).(17).\n",
      "Average loss at step 71: 3.300800\n",
      "\tPerplexity at step 71: 27.134348\n",
      "\n",
      "Valid Perplexity: 78.94\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 70 ... \n",
      "======================== New text Segment ==========================\n",
      " girls so long as the king  , and that that the  wolf which the daughter which  it the son that she  was so long which the  wolf which the son she  she that that she the  old man were so long  so long as the king  which the third which she  that the king and the  little door which the king's  woman was not so long  so the old woman said  and the king which the  old woman said the king  , and the little tailor  , and she that her  other daughter and the great  stone which which the daughter  and the dead which which  and so dead which the  dead one which the other  which which the son which  she that that she was  still long so long and  long so the king and  not not that that the  whole night and said the  king , and the little  girl and the other which  which the daughter which the  wicked which which which the  son which that that that  that the king's woman she  that that the wolf which  that the king which the  little daughter which she all  for her one which the  king's daughter which she the  beautiful daughter which the king's  man and she that the  wolf and the other which  she that that she was  still long for the great  tree and long for her  as she said , i  not not no one be  long as that as that  as the little tailor was  not long so the king  , the little girl was  not very long for the  door which the king's daughter  one which which she and  all dead which she the  wicked which which the daughter  and the dead which which  and so dead which she  all so long and the  old woman which the old  grandmother was so that that  as she said that the  king and not the end  , and that the old  grandmother she she the other  son and that she the  beautiful daughter and the other  one and the other which  which the daughter which the  great which which which which  which the daughter which which  and the other which which  that that that she the  beautiful daughter which the other  which which the daughter and  her one and the other  one and not the other  . and as she and  all all which the great  all which which the daughter  , and that the old  grandmother was so long and  a great gold which which  which the daughter which the  king's daughter which she the  beautiful daughter which she the  beautiful daughter and she she  so that that the king's  man said that she the  king and not not not  it that the old woman  , which she and not  still for it . she  so not that that the  king and the door that  the old woman was not  and not not that that  as not long as she  was so long and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 71)\n",
      "(83).(50).(68).(54).(32).(67).(13).(16).(48).(86).\n",
      "Average loss at step 72: 3.253796\n",
      "\tPerplexity at step 72: 25.888422\n",
      "\n",
      "Valid Perplexity: 81.77\n",
      "\n",
      "Generated Text after epoch 71 ... \n",
      "======================== New text Segment ==========================\n",
      " one of the king , and  she and the other maiden  , who that that the  king were so beautiful that  that he had no other  , and that the third  she the other maiden and  had a other who which  which were so beautiful that  she was so long that  that the tailor which the  beautiful maiden was so beautiful  for that that he were  the beautiful maiden and the  two other maiden and were  a dead which that it  for a mantle which the  little ring which the beautiful  other daughter which were so  little beautiful than the other  princess and the beautiful maiden  which the beautiful daughter which  that that the king which  that the king which the  youngest which the daughter and  all the other and that  that he was not a  end and the little tailor  the beautiful daughter which the  other which were so beautiful  one for him and do  this that the king and  that the king and the  mantle which the other maiden  , and the other maiden  was so beautiful that that  that he were not so  long and he the other  , and the other others  he were so beautiful that  it was so long for  this that he was not  for the end which the  youth which the king and  not the third that the  little tailor was so beautiful  that the king which the  beautiful daughter which he were  to be a beautiful and  life and the little tailor  said , `` that do  that you not like that  . the tailor said ,  and the third i do  you that and my little  wife which you do not  not not not that you  that you do not be  so well that you do  you not do you .  what do you do ,  so that you are not  so long for the third  i do you do to  make that you do not  see that you do not  see you you do not  be so long as the  little tailor and do you  for you , and the  little tailor said , the  third night , and the  youth said , the third  do you not do that  . the tailor said ,  and the third i do  that that that that the  godfather are a king's daughter  one which the daughter which  that the king and the  mantle which that that the  youth said that the third  one which the king's daughter  that she were to be  still so tall that it  . the tailor said ,  `` that do not be  my beautiful wife , and  you do not be my  child . '' and the  next night the tailor which  at last that that the  princess which the daughter and  all the other which that  that the king was so  beautiful that the third she  was so long that the  little tailor which were so  beautiful that that the third  which the king and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 72)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(37).(61).(7).(54).(25).(8).(52).(82).(94).(62).\n",
      "Average loss at step 73: 3.180877\n",
      "\tPerplexity at step 73: 24.067853\n",
      "\n",
      "Valid Perplexity: 79.98\n",
      "\n",
      "Generated Text after epoch 72 ... \n",
      "======================== New text Segment ==========================\n",
      " the old woman said and the  old woman was home and  eat that and do not  be as my wife .  and the king and the  old woman came home and  see that it which the  king's daughter were all and  that the old man were  on a great stone which  all in a stone and  they and that that the  father was so glass which  that the king said that  that the old woman was  not and pass which that  it that that that the  queen was not that the  queen was not for the  wedding . the man was  for all and which the  son which the son which  which the daughter which which  were so dead which the  third which the king's daughter  said , i not not  that you and my father  , and that that that  i not not that my  wife which that that the  king was so dead and  life and pass which the  third and the king which  all in a large which  that it the beautiful wife  come home and pass that  night the old woman said  and the third was the  king's wife , and said  the king , and that  that the king which the  beautiful daughter was a dead  man which were a large  fellow and a stone and  they said , third which  which the old woman was  for a dead which which  it and pass which the  little wife were all dead  , and that that the  father was so glass and  all a great stone which  all which the king's daughter  not not that that the  queen was not that the  father was so glass and  joy that the old woman  had not an end which  that a great stone which  which the son which the  daughter which which which the  beautiful which the son which  which the daughter which which  were so dead which the  third which the king's daughter  said , my father is  for my dead one which  which i not that ,  that i not that my  little wife is a riddle  which you have to pass  for my dear wife and  see and pass that my  dear wife was so glass  that the old woman was  all and which that it  the beautiful daughter and the  little wife was so glass  that the old man was  for all all which the  king and all which the  beautiful daughter which he were  for a dead which which  all for a great stone  where the old wife were  all an glass which the  king's wife said the father  said , i have the  mantle which the godfather which  were so dead which the  king's daughter were all dead  which were a dead and  man and joy which which  the daughter which which were  in a stone and the  king which the son which  at the other which which  all in the forest ,  he that that the old  king said , you are   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 73)\n",
      "(59).(23).(85).(83).(74).(70).(98).(20).(62).(45).\n",
      "Average loss at step 74: 3.133150\n",
      "\tPerplexity at step 74: 22.946154\n",
      "\n",
      "Valid Perplexity: 77.66\n",
      "\n",
      "Generated Text after epoch 73 ... \n",
      "======================== New text Segment ==========================\n",
      " the youth , and the king  which the king and the  king which that it all  not a beautiful man which  were the king's wife the  man which the king which  that the king came and  a great stone which the  king which the king and  death that the king were  not a beautiful and which  it the beautiful man and  the king which the king  which the king which the  king which the maiden which  and the maiden which the  beautiful maiden which the maiden  , and the maiden ,  and the maiden and the  beautiful maiden and be all  all and the king and  that the king were and  not the man which was  the king , and that  that the king which the  king which the king and  all that the king which  he the king and the  man were so dead which  all the beautiful man and  they the king and the  king said , i do  not know that that my  father were , and the  maiden , and the king  's daughter , and the  king which the king which  that the king were so  small and flowers which the  man which the son which  and and the other which  the other and the other  where the maiden which which  which the king which the  king which the king and  all that the king were  all the beautiful maiden which  that a great stone which  which the son which the  son which the son which  that the man were and  cut a little which which  that the king were so  small and each other and  take the king , and  no one . the king  and not that that the  young wife were so long  for the little maiden and  said , you have not  the end . that my  father is so small and  that you not to not  leave it . the king  which the king and all  a little draught which the  other and the maiden which  all the beautiful maiden and  no little dress and the  little maiden . the king  and not not the king  of the king was so  beautiful that the king was  for a stone and the  young maiden , and the  king which the king and  that the king , and  and all all all a  great stone which he the  son which the king and  death the king and not  not that the king ,  which the king's maiden was  once so long and the  stone which the king and  that that the king were  to pass that the king  which the king and the  king and that that the  fox which that the man  came and said , you  will not pass that that  i have not a good  stone and the king which  not the king and the  wedding of the little maiden  which which all all a  great stone that the king  , and that all a  stone and the king which   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 74)\n",
      "(66).(65).(60).(64).(68).(69).(77).(55).(21).(83).\n",
      "Average loss at step 75: 3.252979\n",
      "\tPerplexity at step 75: 25.867290\n",
      "\n",
      "Valid Perplexity: 81.64\n",
      "\n",
      "Generated Text after epoch 74 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " see that the child the daughter  which the son came and  how it no one to  return to the little country  all the other came and  put the other home and  all all all all the  all bed which it the  little light which it which  it the other and the  son the other and the  son which it in it  in the air and the  old man said , the  child came home and not  no one in it and  wine , and the child  which the other and no  more and the other and  which that the other which  which the son which the  son came and the other  , and no one and  the other which the daughter  , and the other and  which the other and no  dead and which it in  his little bed which the  old man came and the  little light which it the  one in its hand and  and all which which the  daughter which the other and  no other and the other  of the other country and  that the child came and  all each other and that  that the king came and  put it and the little  table which the other which  it in the bed which  it the other which the  king which the other and  no one and the other  , and that it the  little castle the other which  which the son and the  mother and the other which  and no other and the  dead which that the other  , and the other and  which no one and the  son and the other which  it in the bed which  it the other which the  king which the son came  to each other and the  two other which which the  maiden which the other and  put it and the bed  and all all which the  one which the other which  which the son and it  in the other country which  it the other other and  eat the little bed which  all all all the little  light which the child which  it the other and the  son the other and the  maiden which that the other  came and return the whole  castle which the other which  the other which the other  and no other and the  daughter which it the other  and the other which the  son which the other and  that the other came and  keep the other which the  other which the other and  no other and it the  son which it in it  and all its little bed  which which the other which  which the other which the  daughter which the other and  no other and the other  in its little bed which  it the other which the  king which the son came  home and put it and  her little bed which the  two which the other and  put all and which it  it in the bed which  it a little which which  it in the earth .  if the mother came and  his little bed which the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 75)\n",
      "(94).(22).(31).(93).(23).(50).(42).(55).(73).(34).\n",
      "Average loss at step 76: 3.157452\n",
      "\tPerplexity at step 76: 23.510603\n",
      "\n",
      "Valid Perplexity: 82.05\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 75 ... \n",
      "======================== New text Segment ==========================\n",
      " by her little house . she  said and the king and  all all all all all  her all all not be  so long and all a  great deal which all all  my little house and all  you do that you are  so long as that as  i have , i have  it for you , and  i do n't know that  for me . what do  what you do that you  will not take it home  or my wife and my  men which all all my  one for them and do  not know that that that  it for you , and  i do n't know that  for me . what do  i go home and you  i do not know that  for me . what do  n't you go , and  what that you have the  king , that you have  not so long for the  old one . that that  that all all all the  very man , and that  that all all all the  very man , and all  all all all all all  once all all all the  old man and it all  all all all the end  which the whole house which  all all all all the  death , and all all  a little and that the  king and all all the  house , and no one  and all all all all  all an end which all  it and all all the  be which all all the  silver one which that all  a little fellow which all  the silver one which all  my daughter . you have  not so long for the  father and do not know  for it . the king  went home and all all  all all all all all  all an end and which  that a little fellow and  that a little fellow ,  if you go home and  do you have to be  a riddle , and the  king and all all your  house , and that that  that the king was all  very long long that he  which all a little and  that the king and all  that the king and all  a little fellow and the  whole house and all my  bread which that you are  still so long as my  little wife and my little  men which all all my  dead which is all a  silver one which that you  for you , i do  what you do that you  will not take it home  or my wife and my  whole house which all your  men and be once so  so long for my little  house and that my father  in the whole world and  that my little house ,  which i not no one  that me to be my  knapsack . i do n't  take it home and my  old wife is all my  riddle and each other and  be my little house and  all you have to be  that my little house ,  or that my father is  for my little house and  do not know what you   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 76)\n",
      "(83).(29).(43).(91).(27).(41).(72).(46).(13).(82).\n",
      "Average loss at step 77: 3.306385\n",
      "\tPerplexity at step 77: 27.286307\n",
      "\n",
      "Valid Perplexity: 78.60\n",
      "\n",
      "Generated Text after epoch 76 ... \n",
      "======================== New text Segment ==========================\n",
      " it in the world . the  next night the king and  his father and that he  was the hedgehog and be  once so long and the  tree and the king which  which the son and that  he the daughter which the  king's daughter that he was  very long long and the  little man and the king  , and that that the  hedgehog which the daughter which  the daughter came and the  father and the other and  his daughter , and the  king and the hedgehog and  his son which that that  that the king was so  beautiful one and which that  all and the hedgehog and  not not the end which  the hedgehog and the hedgehog  which that that the king  and not the hedgehog and  not not the end which  that the king which the  old king , and that  that the king which the  king which that the father  were so long that the  king which that the father  was so long that the  young king , and that  that the king's king was  a little fellow and the  huntsman , and that the  king which the daughter which  he no one and the  son and no one and  his daughter . that that  all the hedgehog which the  king which the daughter and  no one and not that  , and no one and  the king which that that  that was the hedgehog which  that that he was no  beautiful one and the king  , and that the king  , and that that the  king which the daughter came  for the king and how  his father had not the  king's daughter , and that  she the daughter that he  the daughter which that the  king's daughter he was so  long as the king and  the hedgehog and the hedgehog  , and that that the  king's daughter that the father  came and the king and  him that the king was  a little man and the  huntsman which that the hedgehog  which the son which that  that he was no little  one , and that the  father was so great and  white that the king was  very long long one and  his daughter which that the  king's daughter , and that  that the king which the  king which that that the  old king had her daughter  , and that the king  had not a hedgehog and  all the end which the  king's daughter which that the  daughter which the daughter came  and the king and the  king which the daughter and  that the king was a  dead and which the other  of the hedgehog which that  no one and not no  one , and the king  with the hedgehog which that  it and not the hedgehog  and not not the king  with the hedgehog which the  old man which the daughter  and that the king was  a little fellow which the  old king , and that  that the hedgehog which the  king which that the king  had not a hedgehog and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 77)\n",
      "(52).(47).(21).(28).(8).(83).(12).(56).(35).(75).\n",
      "Average loss at step 78: 3.050396\n",
      "\tPerplexity at step 78: 21.123699\n",
      "\n",
      "Valid Perplexity: 79.78\n",
      "\n",
      "Generated Text after epoch 77 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " might see how the little man  which the son was so  dead that the little man  came and put the other  light and the little man  that the king which the  devil which the son was  full which the other which  the other , and that  that the devil which the  son which the youngest which  it and no one and  it for them and see  how the king which the  old woman was so little  that the fox which the  devil which the son which  the youngest which he was  a little fellow and the  tree . the man came  to her and ate and  put and her little bed  his which , and the  little man and the man  lay for the little man  that the man was so  handsome that the man was  a beautiful bird and the  youngest which the father which  the son said , ``  what do you do ,  how i do that my  child which my father have  the king's man , how  do you do that my  dear father was a little  bird and the little man  said , i do not  know that , and have  not a little fellow for  him . '' the mother  went and put her little  little bed and which that  the other which the fox  set him a little pieces  which the devil which the  king and how the king  came and put him and  light the little which the  king gave him a pair  one which he was so  long and put so long  and the little man was  a little white which which  the son and the other  other which the other which  the fox which the son  and no one and the  old woman which the son  said , `` that have  the devil in the world  . '' the mother came  home and put a little  hair and a little white  dress which the fox which  that the man which the  youngest which the youngest which  it and the hedgehog which  that the old man which  a little man which the  youngest which the father and  no one for them for  a little man which the  king which the king which  she the son and how  the king was a little  man and the king and  the other woman which the  king which the father said  , `` that not that  you do the devil and  you have a little fellow  . '' and the man  was so that that the  king came and the other  , and that it the  youngest , and the other  one and him in the  forest and said , ``  how do you do that  that i do not know  how to see my father  , i do not know  that , and have no  human form for my father  that my father have not  such a little fellow and  if you have a little  bird and my little bird  said , you do not   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 78)\n",
      "(57).(44).(62).(97).(89).(3).(93).(66).(38).(67).\n",
      "Average loss at step 79: 3.161758\n",
      "\tPerplexity at step 79: 23.612064\n",
      "\n",
      "Valid Perplexity: 80.27\n",
      "\n",
      "Generated Text after epoch 78 ... \n",
      "======================== New text Segment ==========================\n",
      " , but the king and the  old king which that the  third which that the king  which the king and not  all that the king which  all that the king and  all that the king and  all that the king which  that the king which the  youngest and the king and  the king which that the  old king which the king  was all so that that  that the third , the  old king which the king  which that the king which  that the king and that  the king which that the  king and that that he  came to the door and  that the king which the  king which the king which  that that the king which  that the king which the  old king which that he  all that that the king  which the king and not  not that and was so  long that the king which  he all that the king  which the king which the  king which the son which  the son which that the  beautiful which the king which  all that the king which  that that the king which  it and the king and  all that the king which  he went to the king  , and that that the  youngest which the son which  that the king which that  the king which that him  for the king and the  king which that him and  it that the youngest ,  and the king which the  king which that that he  said the king which the  old king which the king  were all so that that  the king , and that  a little maiden which the  king which that the king  which the king and not  all that the king and  all that the king which  that the king which the  youngest and the king and  all that the king and  the king which that the  king which that him and  no one , and the  old man and the king  which the king which the  youngest and the king and  the king which that the  third which the king which  that the king and the  king which that the king  , and the king which  that the king and not  no one , and the  youngest and that the king  which that the king and  all that the king and  that the king which that  it and the king and  all that the king which  that the king and not  all that the king which  the king which that the  king which that him and  that the king and that  and the king and that  which the king which that  that the king which the  youngest and the king which  that the king and not  not that and was all  dead and all so that  he and the king and  all that the king and  that the king which that  it and the king and  all that the king which  he went to the king  , and that the king  , and the king and  how the king and not   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 79)\n",
      "(84).(34).(47).(32).(28).(1).(22).(40).(89).(94).\n",
      "Average loss at step 80: 2.994909\n",
      "\tPerplexity at step 80: 19.983540\n",
      "\n",
      "Valid Perplexity: 82.67\n",
      "\n",
      "Generated Text after epoch 79 ... \n",
      "======================== New text Segment ==========================\n",
      " as the devil had all an  dead wife which the old  queen which the king and  death that the devil came  to the end , it  that the devil which the  old woman which the third  which all all a little  wife , and that the  old woman came home and  rest which the devil which  all all all all all  once all it all the  little daughter which the old  man which the second which  which all all all all  all all all a little  which the devil came and  put a little light which  that the old woman and  it all all all all  all all all all all  it and all a great  bed which the old woman  came home and put the  ground home and the little  little bed which it that  it and the end which  it all all all all  and all all and the  old and which the third  all all all all the  little daughter which the third  which all all all the  beautiful wife and the old  mother came home and the  old woman which the third  all all all all all  all all all all all  all all all all the  death and and all all  all all all all a  stone and and all that  all all all all all  that the old man which  which all for them and  glass which that the third  one all them all the  devil came and put a  dead little which which which  which all all all all  all all all a little  and which the third the  king came home and put  all a great stone which  which all all a great  be all which the devil  came and put the little  bed which which all all  her all all all all  all all all all the  old and and all all  all all all a little  which the devil which the  old woman came and the  ground . the mother which  that all all all all  no little wife which the  beautiful daughter which which all  and all all all the  little daughter which the third  , and that the third  that the devil came and  all a little bed and  `` that the devil is  all your little wife and  my wife and a little  little meat and my little  mother you and do you  that my father and all  you that you and no  little wife is your little  life and you do not  take your child . ''  she and the third the  second that the old woman  came and the end which  all all a great stone  came and all all all  it all all all all  all all all all all  it all all a little  pieces which all a little  cry which the devil which  which all all all all  all all all all the  death that the devil which  which all all all all  all all all all all  all all a little and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 80)\n",
      "(66).(87).(8).(41).(73).(82).(53).(79).(21).(48).\n",
      "Average loss at step 81: 3.197048\n",
      "\tPerplexity at step 81: 24.460224\n",
      "\n",
      "Valid Perplexity: 82.74\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 80 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " money and the king which the  old woman came home and  let him go home and  fetch the little and the  maiden which the son which  the son which that the  king which the son which  the son which the son  , and the son which  the son which which the  son which which which the  queen which which the son  which which the son which  came and the son which  all the son the son  which which the son which  which all in a little  ring which which the son  , and the son which  which the son which which  the son which the son  that the son came to  her , said the king  , and the king and  son which the son which  it and the son and  that the king which the  son which the son which  the son which which the  beautiful which the son which  the son came and the  beautiful maiden , and the  king which the son which  the son and the son  the son which that the  king which that the son  , and the son which  that the son , which  no one and the king  where the king which that  it and the son that  were so beautiful that the  hedgehog which the son which  which the son which the  king which which the son  came and the son which  which the son which which  the son which the son  the son which which the  king which which the son  came and the son which  that the son which had  a beautiful maiden which which  that the beautiful maiden which  the son which the son  which which the son which  the son which which the  daughter which which the son  which which the son which  came and the son which  which the son which the  beautiful son which which the  queen which the son which  the son which which the  queen which the son which  the son which which the  queen which which the son  which which the son which  which the son came and  no one which the son  the son which which the  beautiful son which which came  so long the king which  the son which that the  king and death the king  which the son and the  son which that the son  , which the son and  the son which that the  king which the son which  which the son which the  king which which the son  which which the son which  were so beautiful that the  beautiful maiden which which the  king which the son which  the son which which the  king which which the son  came and the son which  which the son which the  king which which the son  which the son came and  he the son which the  daughter which the son which  which the son which which  the son which which the  king which which the son  and the son which came  and the son which the  son which that the son   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 81)\n",
      "(90).(99).(56).(85).(73).(3).(95).(37).(14).(83).\n",
      "Average loss at step 82: 3.035036\n",
      "\tPerplexity at step 82: 20.801721\n",
      "\n",
      "Valid Perplexity: 82.05\n",
      "\n",
      "Generated Text after epoch 81 ... \n",
      "======================== New text Segment ==========================\n",
      " my horse and no little bed  , the little tailor which  that it and no little  light which the little tailor  which the son came no  all that it and the  king and no little white  dress and all the little  table which the son which  it and no little bed  and the little bed and  no little bed which the  white maiden which the son  gave him a little light  and its little bed which  the little white which which  all all all all the  night the mother came and  put it and all all  all all which the little  bed which the little maiden  all which the son came  for her little bed which  all all all the little  castle which the son which  that no little bed and  put the little light which  the little tailor which the  daughter came and the little  maiden which the little maiden  which the son came and  no little bed and the  little white other which which  which the son came and  cut the little light which  her little tailor which the  king which the son came  for her little bed which  the little white little white  little flowers which the little  table which the son came  no little bed and the  king and no little white  dress and all which the  white maiden which the little  maiden which the son came  no little bed and the  length which the son which  which the son which the  little maiden came and the  little maiden which the little  night which the son came  for her little bed which  all all all the little  table which the son which  it and no little bed  until the little tailor and  no little bed which the  son which the son came  for her little bed which  which the little white little  night the little table which  no little bed which the  white maiden which the son  were all all and no  no and the little tailor  and no little bed and  no little bed which the  white maiden which the son  which the son came and  cut the little light which  the little night which the  queen and no little bed  , and no one and  it in the bed and  pass the little bed which  all the little night which  which the little white little  flowers which the little maiden  which the son came and  put it and all all  bed and all all the  old little white which which  all all all all the  death which the little tailor  which the son came and  cut the little light which  the little night which which  came no little bed and  seek all all all the  bed which the little tailor  and no little bed and  her little bed which the  queen which the son came  for her little bed which  a little white little flowers  its flowers which the little  maiden which the son which  all all all the little  bed which the little white   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 82)\n",
      "(31).(52).(59).(82).(13).(21).(50).(2).(24).(55).\n",
      "Average loss at step 83: 2.979026\n",
      "\tPerplexity at step 83: 19.668652\n",
      "\n",
      "Valid Perplexity: 79.28\n",
      "\n",
      "Generated Text after epoch 82 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " a little man which all a  beautiful man which the king  of the other that the  third man which the daughter  the son the king which  which the king which the  son which the daughter came  to the great which the  little man which which all  a little and which the  little man which all all  full which that him and  all a great deal which  it so that that all  great gold a great which  which the beautiful daughter which  no little man which the  king which the daughter came  and the one which the  king which all the little  white which which the daughter  came and and the other  one which which all so  all that the king which  all a great deal and  them so that the king  had him a little man  , and that the third  day and all that the  little tailor which the daughter  , and that the king  , and all all them  so a great which the  whole castle which which the  beautiful daughter and all that  the king and all a  great deal which which the  beautiful one which which the  beautiful daughter which all and  put the little which which  all all all that it  as a little which the  old man which all for  each other which the hedgehog  in the other which the  king's daughter which the king  , and the other man  in the other which the  old man which the king  of the other , the  little tailor which the daughter  , and all one which  that a little men and  that the king which the  beautiful man which the daughter  which the daughter which the  old man came and the  two which which all so  that that the king which  all all a great which  all a little which which  all a great which which  that a great deal which  it so a great deal  or a gold which all  all be so one and  that it and be a  great deal which which the  other and which it and  no little and which the  beautiful man which the daughter  was and all a great  ring which the little man  which the king which the  king which the daughter came  home and put the little  which which the other which  that them so long that  the youth which the daughter  , which he and no  little one and it that  a great deal which which  all a great deal which  it so that it which  that a great deal which  it so that it which  that a great deal which  it so that that the  youth said the king and  what the king was not  very long that the king  which all all all all  an great deal which which  all a great deal which  it so that that the  king's king was a little  one and the king which  all a great deal which  it so that it which  a great deal which which   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 83)\n",
      "(94).(79).(1).(73).(98).(38).(59).(82).(48).(63).\n",
      "Average loss at step 84: 2.981436\n",
      "\tPerplexity at step 84: 19.716101\n",
      "\n",
      "Valid Perplexity: 78.07\n",
      "\n",
      "Generated Text after epoch 83 ... \n",
      "======================== New text Segment ==========================\n",
      " at morning and said the father  that the king which not  a great deal which which  my little wife and a  great ring which which the  little maiden which all so  long that the father which  he the son which the  beautiful maiden which the son  and the other and the  son and the maiden which  all so beautiful that he  was so long that the  old man which the son  which the son which the  old maiden which the son  cut his little and which  which the son which the  beautiful maiden which the son  the maiden which which the  other which the son which  all so beautiful that the  maiden which the beautiful maiden  which the son which the  old maiden which the son  cut a little and which  it the other , and  said , i have not  an most man and that  my little wife which that  you are so beautiful that  my father which which not  you that you are not  long long for my father  . the father said ,  i do not know what  you have to be .  and the father said ,  no that i not no  one . the father said  the father , and that  i not not so long  so the king which the  daughter which the son which  which the other , the  maiden which the son which  the son and that that  he the beautiful daughter .  the father said , i  do not know that that  you are so long long  that the father which that  not my father , and  i have not so long  that the king which that  i not not so long  so the father said ,  no , i have not  so long as it which  my wife which that you  i not what you do  that my axe which the  three daughter has not my  father . the father said  the father , i have  my son . the father  thought , i have no  one . that i not  do my father and i  do you that that i  do you know that my  wife which that that that  that the king which not  a great stone which which  my little daughter and the  boy said , i do  you know that you have  a great deal which which  my little wife and that  you are so long long  that the king which that  it which not no one  to be my father .  she said , i have  you for my father ,  and i not not my  end which that you that  that you are not so  beautiful for the king .  hans the hedgehog which the  son and that that he  all no one which the  beautiful maiden which which the  beautiful maiden which the son  lay and and that the  little maiden which which the  king which the son which  no one , and the  other , and cut his  hair them his little which   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 84)\n",
      "(17).(97).(40).(92).(69).(58).(18).(72).(62).(5).\n",
      "Average loss at step 85: 3.187418\n",
      "\tPerplexity at step 85: 24.225797\n",
      "\n",
      "Valid Perplexity: 74.85\n",
      "\n",
      "Generated Text after epoch 84 ... \n",
      "======================== New text Segment ==========================\n",
      " one , and that the wedding  , and the king and  not the wedding , and  said , i have the  old man , and the  old peasant said , i  do not know what that  that that i not not  for my wedding . the  man was not to be  an wife to the king  and the hedgehog , and  all for his wife ,  but the wedding , which  all for a stone which  the hedgehog , and the  third , said the peasant  . what i have to  eat and you do not  want to eat . my  father has not an wife  has a hedgehog and the  wedding . the peasant ,  said that the wedding which  which the daughter , and  he said that the wedding  is a wife , and  do not be an wife  to me . i have  an hedgehog , and that  what that i not not  the hedgehog which you have  no human form and do  that you , i have  a hundred and my wife  shall not be my wife  . that i have the  old man , and the  peasant said , i have  no human form for the  human form and do the  mantle . what i not  what that i not too  for the wedding . the  peasant , said the king  . what i have to  an end , my father  is a hedgehog and not  an end , and that  that that the wedding has  no wife . that the  wedding . the peasant ,  said the peasant , you  are to be my wife  to you . i have  a hundred talers and i  do not know that that  that you are the reward  that my wife has not  your wife . that i  do not know for that  my wife are to pass  that my wedding . the  king , who that no  one , and the wedding  . the peasant , how  that was the king and  his father and the hedgehog  came home and said ,  that that i not not  an wedding . the peasant  which the king which the  king which that that the  king's daughter were to pass  so little little which the  hundred which the king which  that the king were a  hedgehog and the king and  not the wedding . the  man said , that that  are you for my wife  , and i have the  wedding for my wife which  the wedding . that i  do not know that that  that that that i have  an hedgehog , and the  king which i not not  a hundred for the king  with me and i have  it for my wife .  you i not not no  human form and you have  it for my wife .  you i not not no  one . what do you  know what my own has  a wife , my father  said the wedding . the  third time he went out   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 85)\n",
      "(5).(55).(44).(63).(37).(3).(94).(98).(62).(18).\n",
      "Average loss at step 86: 2.997375\n",
      "\tPerplexity at step 86: 20.032880\n",
      "\n",
      "Valid Perplexity: 80.18\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 85 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " who no little coat and which  she the daughter and the  old woman came home and  eat and drink and the  father and mother which the  old woman came home and  eat and meet it and  said , i not not  not my wife has not  a little glass and my  human form and my little  table which my wife are  not and do my wife  have her wife and my  little glass , you are  the roast meat and my  human shape and little stone  which my little wife has  the roast meat which my  good sister which you not  come and do my wedding  for my human form and  eat and eat and my  little glass has not my  roast glass my little glass  shall not be my silver  stone and my little glass  pieces and my little wife  are home and eat and  eat and drink and my  wife are them and my  little wife which a little  glass which that that that  that the little wife were  a glass . that that  the devil came home and  cut the glass which that  that the little maiden were  divided his little and and  the maiden and each other  to pass that that the  seventh , and the little  dwarf said , that that  are you for my little  table which my wife are  the wife , and that  i have not an roast  table my little glass are  all my little glass .  and the father said ,  this , i have not  your wife , and that  that the wedding are not  an wife , and that  that the wedding has not  a little glass or glass  and my little glass has  a stone and that the  old woman said , god  have my little wife are  no little coat and not  my little wife has not  be my little glass and  a little glass which my  wedding which the daughter were  for the little glass which  they said , that that  no little may not not  pass that that the little  peasant said , that that  that the wedding has not  a little glass or glass  the glass glass which my  bread which do not know  you , and that that  my father were and glass  , that that that the  father said , that that  my father , and that  that the wedding which the  daughter which the daughter were  all an little maiden which  she the daughter , and  that the maiden were a  handsome maiden which said that  this the wedding was a  beautiful wife . that that  no little shoes the little  stone and the maiden and  meet the little maiden which  came to the little door  or the wife which the  old woman came home and  eat and eat and drink  or my human form and  see and said , that  my wife are not and  go home and eat and  eat and eat and eat  which my little glass has   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 86)\n",
      "(92).(2).(88).(30).(35).(66).(68).(84).(53).(10).\n",
      "Average loss at step 87: 3.219945\n",
      "\tPerplexity at step 87: 25.026742\n",
      "\n",
      "Valid Perplexity: 80.52\n",
      "\n",
      "Generated Text after epoch 86 ... \n",
      "======================== New text Segment ==========================\n",
      " child . the girl said that  the third which the king  which are you . the  man said , that do  do come , and the  third which the third which  this are a beautiful son  . how do you come  home and do not know  it for you and do  n't know that . the  man said the father ,  do you come and my  king and not that the  king , how do you  do . that do not  do what you have to  my end , you do  not know what my father  said the king , and  the third , the third  and the king and the  hedgehog which the son which  the son which were standing  in the window , and  and the third and the  king which the third which  the king which no and  not the third , the  third , how do you  do for me . the  third , how do you  do so long as you  can not know you for  it and do my end  , you do not know  how your father has a  wife which you come and  do not know what the  king , how my father  not come and do you  shall do my end of  your money . the man  asked him the third ,  you are a beautiful man  , and the third which  you come home and do  my end of money .  the man said , that  come and you do not  want to take my little  pieces , i do not  stir . the man said  , that do not know  how you do not know  it for you . how  go home and do not  do what you do ,  if you do not know  how the third is a  great deal which my bread  to be my wife .  that that that is a  great deal of money and  this my little wife ,  but the father said ,  my father , do not  do what my father ,  i do not know that  it for my human form  and do my wedding .  so the father said and  no one and the third  not the third , and  the third , that do  n't come home and take  a little life and my  mother has a great deal  of money which my wife  and a great deal which  which which you that you  do not know what the  old king , and that  the third said the third  , the third man are  your son and do not  come , and how do  you come home and do  my end of money .  the man said , that  come and you do not  what you do that .  how do you come and  a little man and a  little man is a dead  man and which my wife  are a great deal which  the roast meat that the  second which the queen which  not that the king ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 87)\n",
      "(34).(79).(38).(41).(65).(7).(15).(11).(49).(63).\n",
      "Average loss at step 88: 3.354131\n",
      "\tPerplexity at step 88: 28.620718\n",
      "\n",
      "Valid Perplexity: 77.56\n",
      "\n",
      "Generated Text after epoch 87 ... \n",
      "======================== New text Segment ==========================\n",
      " , he and the king and  his daughter , and the  old man came and the  little maiden and the man  said , i do not  see that you are not  my wife . and the  man said , i do  not know what you have  for you and do you  come and do not know  what you have , i  not not not my wife  to eat . you do  my wedding . the king  which the king and the  king and the end which  he the other , and  how the man the son  , and that the other  which the son which the  daughter and and a great  white which which which the  other which the son and  and the man which that  that the man were so  white that the king said  that the king and the  king and the king and  his wife were dead and  eat the little which which  all a great stone ,  she and that the king  also and no one and  his wife was dead and  said , i do not  see that i have no  human form . you have  no little life and you  be my wife and my  wife and my wife and  my wife which no little  daughter which the daughter ,  who the son and the  old woman came home and  cut the little which which  that a great stone which  no one and the man  , and the man and  a little man and the  man and that the man  and that the king which  that the king said the  daughter , i have no  wife and my wife and  the little meat which my  bread . and the man  answered , i do not  see that i do not  know that that i not  not my wife , i  not not not know that  my wife has not a  great stone , i not  how that you , i  i not how that you  so long and my wife  which my wife are a  roast meat . you and  how you do not know  my wife , you have  not my wife , i  will not not not know  my wife . the man  said that the king said  the king , i have  my daughter . but the  man said , i do  not know that that i  have a great deal which  my bread . i do  my wedding . the man  gave him a great stone  and the man said ,  i do not know what  i do , and you  will not be my wife  for my human shape and  stone and i have a  little deal which my wife  are a little meat and  a great stone which the  little wife has a little  stone and the little bird  said , i do not  see that i do not  know that that i not  not my wife . how  that i not not my  wife , i not not   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 88)\n",
      "(13).(87).(38).(73).(78).(48).(5).(53).(96).(33).\n",
      "Average loss at step 89: 3.104511\n",
      "\tPerplexity at step 89: 22.298301\n",
      "\n",
      "Valid Perplexity: 79.52\n",
      "\n",
      "Generated Text after epoch 88 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " as you have , i do  not know that that the  master is a great deal  for them and bring it  a little apple which my  little meat and a great  stone , i have a  great apple which you have  a great deal which which  my wife . the father  and son and that the  father were so beautiful that  he were not so so  a long long long which  which a great ring which  that the son which the  son which the son which  which the son which that  he the son and the  little maiden which that the  king which the father which  that the father which the  daughter which the son which  which the son which the  other which the son which  the son which the son  which the son which that  he were a dead which  it in the great forest  and which the king which  she the son which the  son which the son which  it at each other and  set him home and said  , i do n't know  what you have , i  not not know what you  have , i have a  great deal which which your  money , or that i  are so beautiful that that  that i have a great  deal which which my wife  , i not not know  my father , i do  not know that you are  a great deal which my  son which my wife are  them for my father ,  how do you go and  do you go home and  see that you are so  that that i have a  little deal which which you  that that you are so  dead that you are so  dead that you are so  dead that you are so  so long that the father  answered , you are to  go home . you are  so beautiful that you are  still dead for my father  which i have the great  apple and you have the  little meat and my little  deal which which my wife  . what that i are  there , and you have  a great apple which you  not for a great deal  , and that you are  not so beautiful than that  this are you for my  dear wife and you have  him a little apple and  a great deal which which  your little meat and a  little deal which which which  all a great deal which  it which the father be  be so that the little  bird said , i have  a great deal which it  your wife . what that  i not not a great  box which you have a  hundred which you are so  beautiful that you are so  so long that the father  answered , you are to  go home . you are  so beautiful that you are  the most beautiful man ,  do you go home and  eat . you are to  sit home . you are  not your little meat and  long so that you are  so beautiful that you are   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 89)\n",
      "(69).(60).(99).(92).(73).(32).(43).(78).(28).(37).\n",
      "Average loss at step 90: 3.108054\n",
      "\tPerplexity at step 90: 22.377460\n",
      "\n",
      "Valid Perplexity: 80.67\n",
      "\n",
      "Generated Text after epoch 89 ... \n",
      "======================== New text Segment ==========================\n",
      " afterwards a little man which the  old woman came home and  put her and and the  two which which that the  old woman and that that  that the queen were a  old man and that the  king which the queen was  that that the queen ,  that the queen would not  have nothing and the king  , and the king which  that the king and death  that the king which no  one and the queen which  the queen , and the  king and death that the  king which the queen were  the queen and that she  came to the country and  happiness and the little woman  said , that is a  old man and that i  i do not know that  the little woman has not  an old man and the  queen and that you do  that that that that i  not not a little man  . that that that i  do not know that you  do that you have the  queen and that i have  the old woman and the  king and that that she  are the king which the  father have not that the  queen which the queen has  her son and the king  , and the queen and  the king which the queen  which that the queen was  a little man and the  king and that the king  , and the queen and  that the king which no  one and the king which  that the king which the  old woman came home and  meet the little man which  which the son and the  old woman came home and  put her and which the  king which that the man  which the queen which the  old woman came and meet  a little little man which  which the son which the  king which the son which  which came to the end  where the old man came  home and put her and  so that the little woman  which that the queen came  that the king which the  king which the queen was  that that the queen ,  which the queen which the  king which the queen and  that the king which the  king which that the queen  had the old woman and  that the king and death  for the third time and  if the king which no  one and the king and  the king which that the  little man came home and  thought the queen , that  the queen is not long  . the queen , that  it the queen and the  king which the queen and  that the king , how  this that that no one  which the queen which the  king and death that the  man would not find it  . the man and the  king which that the queen  came and the little woman  she that that the king  , and the queen ,  that the queen , and  that the queen which the  old woman came and the  one and the king and  that that she the queen  and the king and death  for her . the man   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 90)\n",
      "(58).(99).(45).(33).(23).(14).(3).(40).(25).(36).\n",
      "Average loss at step 91: 3.135179\n",
      "\tPerplexity at step 91: 22.992753\n",
      "\n",
      "Valid Perplexity: 82.62\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 90 ... \n",
      "======================== New text Segment ==========================\n",
      " huntsman and the king and the  old woman ran home and  cut a little which which  that the man and the  old woman ran and the  old woman ran home and  put a little which which  no one to eat and  drink , the man ran  to the door and the  old woman ran home and  let her go home and  drink and the wolf were  cut a little which and  it and no one to  eat . the man ran  so that the wolf were  white of great which the  poor woman ran and the  old man and it in  her own house the tree  full which which were a  beautiful maiden which it had  so long that the little  beast and the wolf ran  so that the wolf which  no one and the man  , and that the wolf  which the wolf which the  old woman and one of  his father and the old  old woman ran out and  cut the tree and the  old man and it was  just as it for the  wolf and said and no  father , i not not  a fine trade . the  tailor , and that as  she was so long as  a little dog and the  man and the old woman  began to cry and the  old woman and it in  her hand the little dog  or the king and the  old woman ran home and  let her go home and  bring her to the tree  , and the wolf ran  home and cut a little  little which which it and  for the old woman which  a little tailor , and  no one to be a  wife , the wolf ran  him and no one to  get home and eat and  eat and drink and the  tailor and the king which  it was so long as  a little dog and the  king and the king which  that the man which the  old woman ran home and  put a little which and  the little maiden which the  old woman ran home and  put a little which which  the old which which the  old one and the old  man and the old man  ran home and cut a  new and the tree which  it and no one to  eat . the woman ran  for the little man and  it at length and the  wolf and it at one  arms and thread and the  old man ran home and  let her go and see  the little tailor and the  little tailor which was a  great deal which which were  full which that the man  , and that it was  just only not not long  so that and thought the  old woman said , i  not not not not no  so long as it for  that and said , what  is to be my wife  , i have not a  handsome one . the wolf  said the wolf , who  no one has not my  little wife and the man  which had not no one   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 91)\n",
      "(96).(20).(72).(84).(26).(56).(95).(46).(68).(67).\n",
      "Average loss at step 92: 3.283686\n",
      "\tPerplexity at step 92: 26.673903\n",
      "\n",
      "Valid Perplexity: 78.87\n",
      "\n",
      "Generated Text after epoch 91 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " said the king , and that  you do not know that  . the king said that  that that he were a  white daughter , and that  he were not so long  that that that the king  and not that the king  which he were to be  so great a great stone  , that that the king  was a king which the  king which he were to  go home and put the  pair which that that that  he were a king which  the king which he were  there , and that the  fox which the king which  he were to be an  king and the king which  he were to be a  king which the king which  he were to be a  king and the king which  that he were a king  and the king which that  they were a king which  they were to be a  king which the king which  she were to be a  king and the king which  he were to be a  king which the king which  he were to be a  king and the king which  they were to be a  glass which the king which  that that that the king  which he were to be  an king and the king  , that that the king  , and that that the  old man said that the  father said that the king  said that that the king  which he were to be  an king , and that  the king which the king  which were in a great  forest which the king which  that he were a king  and the king which that  he were a king which  he were to be a  king and the king which  that he were a king  and the king which that  they were a king and  that that he were a  princess , and that that  the king which he were  no one , and that  that that the king were  once dead and the king  and the king which that  they were a king which  that he were a king  , and that that he  did not know that ,  but that that the king  , and that the king  was a king which the  old man said that the  third which the king which  he were to be an  wife and the king and  that the king which the  fox were the king and  that that the king which  that he were a king  and the king which that  they were a king which  they were to be a  king which the king which  she were to be a  glass which the king which  he were to be a  glass which the king which  he were to be a  king which the king which  he were to be a  king and the king which  that he were a king  and the king which that  they were a king which  they were to be a  king which the king which  she were to be a  glass which that the king   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 92)\n",
      "(92).(36).(25).(72).(90).(34).(10).(11).(7).(64).\n",
      "Average loss at step 93: 3.114442\n",
      "\tPerplexity at step 93: 22.520857\n",
      "\n",
      "Valid Perplexity: 80.17\n",
      "\n",
      "Generated Text after epoch 92 ... \n",
      "======================== New text Segment ==========================\n",
      " of it , and the queen  said the queen , and  that the queen had the  queen daughter , and that  no one , and the  little maiden said , how  do my child , i  do not know what the  king , and the queen  would not come home and  said , that no little  ones . the queen had  the king and the king  also and no one was  not no one to be  very long , and the  third , the queen ,  and the king and the  son which were in a  bed which the queen which  they the king and the  old woman came home and  said the king and the  second , the queen ,  said the queen , how  good i do that you  do not know you ,  i do not forget you  , and i not not  come and do not know  for the king , and  it will not be a  king , and how not  my wedding , the queen  said the queen , and  that the queen had the  queen daughter , and the  queen had the king and  the king and the king  and the king which the  old woman came home and  cry and the king and  no one , and the  queen , and that she  was to see how the  old woman had not a  wicked maiden , and the  queen , and the queen  had not come to the  door . the queen said  that the king were so  beautiful , and that the  old woman had the king  and the king and the  princess which had to come  with her and the king  began to weep one and  the king and the king  went and and the king  , how that the queen  had not come to the  king , and the king  and the king and the  king and the king and  no one , and the  king and not not that  she was to be a  king , and the king  , and that that she  were to see how the  old woman said , how  good you shall be my  wedding . the queen said  that the queen were all  so beautiful that the queen  , and the queen was  a king , and the  king and not come ,  and said the queen and  she said the queen ,  what not my wedding which  no one , and the  king and the king and  no one , and the  king and not not that  that the queen , and  the king , and the  king and the king which  no one and the king  which the queen were to  carry her . the king  also the son and the  queen and the maiden which  the son which the son  came and the king and  no one , and the  king and not not that  , and said , how  good you shall not be  so long that you have   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 93)\n",
      "(89).(88).(17).(98).(58).(8).(59).(54).(85).(7).\n",
      "Average loss at step 94: 2.962005\n",
      "\tPerplexity at step 94: 19.336694\n",
      "\n",
      "Valid Perplexity: 79.18\n",
      "\n",
      "Generated Text after epoch 93 ... \n",
      "======================== New text Segment ==========================\n",
      " follow us . the king answered  that the king and the  son which had a great  cry , the queen and  that the queen was a  queen and the queen which  she the son which the  king which the son which  all and the queen which  had the king and the  queen was so long that  the queen which the queen  , and and the queen  and the king which the  daughter which the son which  it and the son which  the son which the son  came and the queen ,  and the queen had her  daughter , and the queen  was so long that she  went and cut the hill  where the queen which the  daughter which the son which  the son which the son  and the queen which the  daughter which the son the  daughter which at the king  , and she went and  and the king and the  wife went home and cut  a little life and the  old woman was so long  that the king and all  that the king and the  daughter the son and the  maiden which the son which  the son which the son  came to eat and fetch  it , the king and  this the king and the  queen , and the queen  was so long that the  queen which the king which  she the daughter which the  son the queen were quite  there , and and the  maiden and all all the  king which the king and  how the king and the  queen were so long and  went to the door which  the king which the son  and the queen , and  the queen had the king  , and the queen had  no wife and the king  until the queen was all  his wife the queen and  his wife began to cry  until the child which the  queen which the son which  it and the son and  the king which the son  and the king which the  king which the son the  daughter and the queen were  once so long and the  king and the son which  which the son which the  queen which the son which  it and the son and  the king which the son  all and the king and  the son which the son  and the queen which the  daughter which the son which  the son which the son  and the queen which the  maiden which the son which  the son which the son  came to eat and fetch  it , the king and  not not that the queen  which was a king and  eat and fetch her the  cry , the king and  his son which the son  the son which the son  and the king which the  son which the son which  the son which the son  all and the queen which  was so long the queen  that the old woman went  out and cut the hill  where the king which the  king which the son which  the son which the son  came to eat and fetch   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 94)\n",
      "(34).(98).(89).(33).(97).(77).(79).(32).(42).(86).\n",
      "Average loss at step 95: 2.966307\n",
      "\tPerplexity at step 95: 19.420065\n",
      "\n",
      "Valid Perplexity: 83.37\n",
      "\n",
      "Generated Text after epoch 94 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " went to church and the tailor  , and the little tailor  came and the tailor and  that the little tailor was  not the king and the  king and the king and  his wife said , ``  what do you be my  little ones . you are  not as long as it  shall be an end ,  so that the tailor is  and my little tailor ,  and the tailor said ,  no , i have not  as long as you do  and do you do .  what do not be my  dear wife , and the  little tailor said , ``  i do not know that  that i have not so  thousand a new one .  if you do not know  how to be my wife  in the greatest world and  all my little ones .  if i not not that  and do not be my  father , and the tailor  said , the tailor is  the little tailor , the  king said , `` that  i do not know you  , and do you know  . '' the tailor said  , `` that i not  see you and see that  i do not know that  and do you have a  handsome end , the tailor  answered , the tailor is  to do and do you  do . '' the tailor  said , `` that is  this and do you know  for the little one and  not be a end which  my daughter . i have  nothing and the end of  it . '' the tailor  took a little which it  the daughter . and the  little tailor said , ``  if you do not know  that , the tailor is  and my little tailor ,  if you have not an  end , and the tailor  said , the tailor is  your little ones . the  little tailor has the end  home . '' the girl  said , `` i do  n't know that , the  child is in a great  bed and be my wife  to be my wife .  that i do not know  that , the tailor is  there . '' the tailor  said and the tailor ,  said the tailor , the  king have the fox you  not be my wife ,  and the tailor said ,  `` that do not be  my wife , and the  fox do you know ,  if you do not know  how to be a little  wife , and the tailor  would not see you and  eat the earth and the  tailor and the little tailor  which was to be a  handsome end that it must  see that the little tailor  came home and put a  princess home home and the  king and the king and  his daughter to eat ,  but as that as the  tailor said and no little  tailor , `` i have  nothing for you and see  you if you be my  child , if you do  be a end , and  if you do not know   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 95)\n",
      "(64).(59).(97).(73).(6).(30).(62).(29).(15).(57).\n",
      "Average loss at step 96: 2.999933\n",
      "\tPerplexity at step 96: 20.084199\n",
      "\n",
      "Valid Perplexity: 77.32\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 95 ... \n",
      "======================== New text Segment ==========================\n",
      " but it is not so long  the long long and and  this as long long long  that the king and the  wife were all so long  a great which were so  long that the king and  the king and the son  said , i have the  bride the king and not  not that the king and  not long that and said  , i not not long  that . the king said  , the father said ,  my father , i not  go home and eat the  hand . the king and  the son said , that  that i not not so  soon as the king and  death for the little table  . the king and this  he the son which were  very long as the king  , and the king and  that the king were all  very long as long as  the king , and the  two maiden whom the maiden  came and the king and  the maiden and the king  which the maiden were all  behind the maiden and cut  his life . the king  , and the maiden and  the son and that the  old woman came home and  cut her life and the  two maiden which the maiden  went home and cut her  hand and the king and  that the king and the  beautiful maiden came home and  cried , the king and  what i not , and  have not for my wife  to eat , and that  that the king and the  son were so beautiful that  the king which the son  came to the table ,  and went and cut the  two maiden and the maiden  and the maiden and the  two maiden which the maiden  and how the king were  so beautiful as the king  with the king and death  . the king said ,  i not not that .  and the king and the  wedding of the maiden and  the maiden and the maiden  and the king and the  son were so beautiful that  the maiden were so beautiful  that the king which had  the beautiful maiden home the  maiden and son and the  old woman said , that  are you so long that  my wife have not so  use . the king said  he was to be an  old man , and the  father said , i not  not long long long long  for her so long long  long that the king and  no wife , and the  king and how the king  was so beautiful that the  old man were so beautiful  until the king and the  old woman said , that  are you for you and  my son and the king  with the king , and  the king and this that  it was so beautiful that  this the king said ,  that is not my wife  wife . the king and  that the king and the  son were so beautiful that  this the king were all  an beautiful maiden and the  two maiden and the son  came home and cut her   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 96)\n",
      "(51).(61).(92).(17).(60).(47).(70).(99).(79).(40).\n",
      "Average loss at step 97: 3.109317\n",
      "\tPerplexity at step 97: 22.405729\n",
      "\n",
      "Valid Perplexity: 81.26\n",
      "\n",
      "Generated Text after epoch 96 ... \n",
      "======================== New text Segment ==========================\n",
      " UNK . she said , what  have you not my wife  ? you have not a  fine fine trade . ''  and the king and the  king and the king which  the son and the king  which the son which were  very long so long as  she said the king and  have the king , and  that that the king which  i not have the king  , and not be a  fine fine trade . and  a little girl and the  old woman and the king  's son which the son  and the son and how  she the king and death  the king and how the  little woman came which a  man came and the king  and the king and the  king which the son which  came to her little bed  . the king said ,  no , i have not  . do you have to  be my wife . the  second day and the king  and death the king and  death that the king and  that the king which the  beautiful woman which were sitting  so long as it and  not long that and was  a king , and the  maiden and the king and  him that the king ,  what the king which are  the king , and not  do that that have you  do so that you have  have the king and not  be my wife to be  my wife . the girl  and the king and the  queen 's daughter and the  second that which had a  handsome man and the king  which the king which the  son which were all a  heap which which the king  also and the king and  all the king and death  that and the king and  that the king were all  so long and long as  it as long as it  which the king and death  that the king which the  king were all all and  and the king and death  that the king and the  king and death the king  who how that the king  would not have an king  with gold the king and  death the king and death  that that the king which  had the king and the  white white man which which  the king which had no  little bed and see that  it the king and not  see it . the girl  which the son and the  maiden which the son which  came to the door which  the king which were all  an king and death the  old woman came home and  said , what do you  have . you have a  great stone and the king  , and that that the  little dove , and the  third time and the king  and how the king were  all all and which the  king which were all a  heap and the gold which  she the daughter . the  woman , however , had  not the king which the  daughter and that the king  , son which was not  long as the king and  come home . she had   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 97)\n",
      "(57).(6).(67).(46).(49).(11).(95).(85).(94).(78).\n",
      "Average loss at step 98: 3.310249\n",
      "\tPerplexity at step 98: 27.391955\n",
      "\n",
      "Valid Perplexity: 79.67\n",
      "\n",
      "Generated Text after epoch 97 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " for his wife , and the  king which the son which  the daughter which the son  the son which were all  very so that it and  not that the man were  all all and the king  and death that the man  came home and said the  king , and that that  i not not the dragon  which are a great stone  and the king and what  you do that you are  an wife , and that  what that that you have  the king , and that  that that that the king  are all dead and do  what that that you have  the king which are a  little meat and a great  cry which the king which  that the woman were all  all so long as the  little white horse and the  old woman said , i  do not know what that  do you have a great  man which you have a  stone and you do not  know that that you have  the king and not not  the man , and that  that the king and not  the man , and the  son and the king which  that the woman came and  be all and the king  , and that that the  king which the king and  what that the man came  for his wife and the  king and what the king  came , and said that  they the king and no  little white dress which the  daughter which the son were  once dead , and the  king and death that the  little white horse came and  set him a great stone  . that that that no  man are the king and  come home home . the  woman said , that is  not so no long that  i do not know what  that do you have a  pair of bread and be  glass and hunger which you  come home home and do  that you do not know  it , i have no  good money . that that  your wife are all a  great stone , that that  that you do not know  what that do you do  not know for the man  , i have a little  draught . the man said  , i have a great  dress which you have the  man which you have no  little white dress which you  that that you are a  white white dress which it  and the man and the  king and what no one  is a little meat which  my wife have a great  dress and a great stone  who which that that you  have the king and not  the man , and not  the man , and the  man and the woman which  the son which were all  very dead for a great  glass which which were all  so long as it and  the man which the king  and death , and that  it the king and no  long money , and the  old woman said , ``  what do you do ,  but you have a great  fire which you have the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 98)\n",
      "(0).(13).(80).(21).(66).(74).(84).(25).(72).(60).\n",
      "Average loss at step 99: 3.339795\n",
      "\tPerplexity at step 99: 28.213330\n",
      "\n",
      "Valid Perplexity: 79.03\n",
      "\n",
      "Generated Text after epoch 98 ... \n",
      "======================== New text Segment ==========================\n",
      " they set him home and fetch  it , and he went  to the door , and  the king said that he  came to the door ,  and when the third man  said that he was to  go home and fetch it  . he went home and  cut a knapsack and the  golden door which had to  sleep . and that the  king , however , would  come home home and said  , i have not no  use for me for it  for you . i have  the king , and that  you have not for a  stone , and that i  do not know that ,  and that i have no  one , and that you  come home home home .  he said that no one  would not get home and  eat it . the king  said , i have not  it for my father and  not not not that it  as a long time and  i not no one .  but the king said that  he was to be a  long time , and that  that he would not be  as so long as it  for him and said the  young king , and that  me to eat , you  shall not find it .  if you do not know  that that i not not  . i do not know  what that have no one  ? '' and that the  old king said , i  do not know that ,  i not not not not  you know that that you  shall be my daughter ,  that me , and not  the king , and not  that that i have a  long time , and that  the knapsack , and the  king said , i have  the king , and that  it is not my wife  . the king said ,  no , i have no  use , that is not  no one , and that  that you shall not be  as well . '' the  king said that he had  an king , and the  old king said , i  do not know that ,  said the huntsman , i  come home , and the  old king said , i  not not not not that  it . i have no  father of them , and  that i have not a  little fellow which you not  not for it , i  not not not not that  that that you have no  one , and that me  , i not not not  it for me , i  do not know that ,  but that i have a  little fellow and be my  silver wife and you have  an king , and that  you shall not be my  wife , and that you  have not for my father  and i not not not  that that that i have  the king , and that  it is not your father  , and not not that  you shall not be so  my than that that it  your wife , or that  it is not my wife   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 99)\n",
      "(22).(35).(77).(13).(60).(63).(43).(8).(33).(21).\n",
      "Average loss at step 100: 3.153600\n",
      "\tPerplexity at step 100: 23.420236\n",
      "\n",
      "Valid Perplexity: 82.53\n",
      "\n",
      "Generated Text after epoch 99 ... \n",
      "======================== New text Segment ==========================\n",
      " . he said that the devil  came and and the little  tailor which he had to  pass , but the devil  would not know what the  old man came to the  place which the devil which  the king and son and  no one , and that  that the devil had no  one sleep for his wife  . the devil and the  man and the king and  what the king came home  until the devil which the  old woman came and see  what the devil came and  and the king and the  other woman which he came  so long so long and  as the devil came and  and the king which he  came to the country and  which which he he in  a great forest which he  came to the door ,  that the devil had not  come out and weep and  all a great stone which  were so beautiful that the  devil came and and the  little man which he came  to the town wherein the  old man which he came  so long so long and  as the devil came and  and the king and all  which he came to the  tower , and the king  and what the king came  to the end which which  came the king and son  for the king , and  the king , and the  king and son and the  man which he came and  cut his life and the  king and what the little  little house which he had  to eat . the devil  let him get home and  cried , i not not  you that too than that  me , and the devil  answered the king , and  she said , i not  be my wife , and  it is not my wife  and you be my wife  for me to hell .  if you not the little  tailor , and the devil  said the king , and  the king which had no  like that the devil would  come and see that the  devil came and and the  king said , i not  not know that , said  that , i have the  cause which the devil has  set him so gold and  you do you know that  my wife is dead ,  i not not stay my  wife , and the devil  said , i not not  my wife . you are  my child , and that  the devil is not a  great stone . i have  the devil in the world  . you have the devil  and not that , answered  that , the devil would  come out . the devil  said the king , and  the devil has set him  home and see that you  can not be my wife  and you be my wife  . the king said ,  that is not my wife  , and the devil said  he , and the devil  said the devil , the  old woman said , i  do not know how to  be my wife . the  king said , what is   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 100)\n",
      "(96).(16).(20).(60).(8).(40).(74).(36).(18).(6).\n",
      "Average loss at step 101: 3.091596\n",
      "\tPerplexity at step 101: 22.012181\n",
      "\n",
      "Valid Perplexity: 79.40\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 100 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " on a great glass which the  devil which the king and  the daughter which she was  so beautiful that she had  no human form and a  man and the king which  it was so long the  woman and the king and  the king , and the  maiden with her her daughter  which the king which were  over , the maiden said  , what do you have  to not that that you  do not be my wife  until that my wife are  no one to be a  fire . the king and  that the king was all  an beautiful maiden and the  woman which the maiden which  came to her little house  the little maiden which were  so beautiful and this that  he was no one ,  but the king and the  king and that the king  came home and said ,  what do you do .  what do not be too  fine for my wife and  not be my wife .  but the king and this  all a great glass which  one one and the king  which the maiden which were  in a great glass which  that that the king was  a beautiful man and the  maiden , and the king  and that the king came  home and said that the  devil which the king which  came to the door which  which had no human form  . the king and this  that the king and not  come . the king ,  and the king , and  the king , and that  that the king which had  no wife to be a  man and the king which  that the king which were  all the king which the  little daughter came and the  maiden , and the king  of the king and the  king and that the king  , and that the king  which the king which were  empty , the king ,  and the king , and  the king , and that  that the king which had  not no human glass .  and the king and this  all a great glass which  the king and what the  king which had no human  rest . the king and  all that the king which  that the king came and  his little wife came home  until the king and the  last one of her little  wife which the king which  the son came and the  queen and the maiden and  how the king which had  her daughter , and that  was so beautiful that the  king was all a great  noise and which that the  maiden , and that the  king which the king and  the daughter which she was  in the forest and which  she was so beautiful and  the king and the king  's son and the king  of the king , and  all the king and the  daughter , and the maiden  , and the king ,  and the king , and  the maiden , and that  that the king which were  a little daughter and the  young maiden and the king   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 101)\n",
      "(70).(99).(79).(72).(54).(51).(41).(5).(13).(87).\n",
      "Average loss at step 102: 2.941867\n",
      "\tPerplexity at step 102: 18.951190\n",
      "\n",
      "Valid Perplexity: 80.26\n",
      "\n",
      "Generated Text after epoch 101 ... \n",
      "======================== New text Segment ==========================\n",
      " a golden apple , and the  king , and that it  was not long as the  little sun , and the  peasant , however , had  a great apple and the  golden apple which was no  one life to eat and  bring it to light .  and the king , how  the king , how not  it to be my wife  , and the sun were  so long as the whole  whole world which the king  's daughter which were all  for the king , and  do not know how to  tell it , and the  king , and the king  , and the king ,  and that the king would  what come home and said  that no little joy which  should not bring it to  sleep . and the sun  came home and said ,  no , i not not  it to be my wife  . the king , how  not a fine trade ?  that , and not not  my wife , and not  a fine apple , and  the king , and the  king , and that the  youth said , that is  this that i have no  human work . what your  dear wife , and not  do it , and the  peasant , the woman said  that the king which had  to bring it to light  for the whole world and  that it and all a  beautiful apple and which the  beautiful daughter which had no  human work . the sun  had so long that the  old woman said , that  are you a fine fine  trade and my wife .  and the sun would not  love it to her and  and the little town which  they would not tell you  . and the sun said  , what do you do  , and not not my  wife , but not not  not a fine fine fine  favor and which which the  king and death and and  love in the world and  bring it to light .  the sun said , that  do not be my bride  and not not as it  and bring it to light  . and the king ,  however , should not know  that , said the peasant  which the king , what  not a fine fine trade  which my wife , and  not not a fine fine  ravens and the apple ,  or that the king ,  however , should not bring  it to light . and  all and said , what  no that we have no  little work and not my  wife , and not not  know that , said the  little hare , what not  my wife , and not  do it and bring the  tree home and bring the  whole apple , and as  this as the whole world  said the king , what  no that do not be  my bride , and the  sun , and the sun  were so long that the  little hare said , what  is that that i not  not as long as you   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 102)\n",
      "(39).(74).(92).(91).(80).(93).(33).(38).(10).(23).\n",
      "Average loss at step 103: 3.115401\n",
      "\tPerplexity at step 103: 22.542466\n",
      "\n",
      "Valid Perplexity: 89.54\n",
      "\n",
      "Generated Text after epoch 102 ... \n",
      "======================== New text Segment ==========================\n",
      " might take her and and the  first table , hans ,  but the king and what  the king , what do  what do you do .  i do not know what  that do . what do  nothing do what you do  . what do you do  . what do you do  . you do not know  how you do . what  give you me ? my  little girl , hans .  i do not know what  i do . what do  you do . you do  what you have . what  i have nothing . what  i do not know what  i do . what do  what do you do .  you do not know what  i do . what do  what do you do .  you do not know what  i do . what do  you do . you do  you know . what do  what do you do .  how do you do .  i do not know what  i do . what do  not do . what do  you do . what do  i do . what do  what do you do .  you do not know what  i do . what do  you do . you do  what you do , and  not not that , and  do you know . what  shall you do . what  my father do not know  what that do you have  . you do not know  you . hans . what  not do you do .  what do you do .  you do not know what  i do . what do  what do you do .  you do not know what  i do . what do  you do . you do  what you do , and  not not that , the  tailor , what do you  give . what do you  come . what do you  do . what do you  come and my wife and  you . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  give . me , hans  that you do not know  you . hans . what  have you my wife ,  what i do . what  i do not know what  i do . what do  what do you do .  what do you do .   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 103)\n",
      "(29).(78).(21).(2).(91).(90).(54).(71).(88).(24).\n",
      "Average loss at step 104: 3.273271\n",
      "\tPerplexity at step 104: 26.397543\n",
      "\n",
      "Valid Perplexity: 83.27\n",
      "\n",
      "Generated Text after epoch 103 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " boy , the king was so  beautiful that the king which  no one were so long  that the old woman were  a great deal which which  one one and the three  wife and all and the  daughter which he was to  see that , the king  and not that , the  woman said that the king  had him , and that  the king , and the  old woman was all on  it as long as it  as it as that the  king which had to be  hold as all as the  king and not that ,  and that the king was  so handsome that the king  was so handsome that the  old woman were all so  very long that the old  soldier said , `` that  that that no one are  me , do not know  it and the king which  would not be so very  long as it as that  it as all as the  devil said , `` that  that that i not not  not as fine as as  long as it as that  your wife . '' the  king said that the king  and the king , and  the king , and the  three man , and the  daughter , and that the  maiden was so handsome that  the king which had to  pass that the king was  sitting as all as it  like the king . the  third night the king which  he the daughter , and  that that the king which  the daughter which were so  son that the king was  so long as the old  man which the daughter which  the daughter which the daughter  came and and that the  maiden was so long that  that the king which had  for the old woman which  the daughter , and the  king , and that that  the king which had to  die , and that the  third , and that he  would not know what the  king which had to be  all on his bed and  and the king , and  and the king , and  that that he was so  very long as that the  old woman was all on  her bed and the king  , and the king ,  but that the king was  a handsome man , and  the king , and the  three man of him to  light , and that the  devil was so long as  the old woman which the  daughter , and the king  's son and that the  devil was so long that  it as all as the  old man , and the  two man , and the  king , and the king  , and that the king  was so long as the  king and not that ,  he that the king was  very handsome that the king  , and the king ,  that the king , and  that the old woman was  so handsome that the king  , and that the king  , and that the old  woman was all in his  way and all all a   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 104)\n",
      "(55).(58).(43).(80).(18).(9).(37).(93).(53).(79).\n",
      "Average loss at step 105: 2.946520\n",
      "\tPerplexity at step 105: 19.039572\n",
      "\n",
      "Valid Perplexity: 86.06\n",
      "\n",
      "Generated Text after epoch 104 ... \n",
      "======================== New text Segment ==========================\n",
      " . the child says , i  do not not know for  my child , have it  for my wife , not  have not so fine for  my father . the paddock  replied , the child says  . the paddock says ,  no , i have not  for little deal which my  queen which my wife which  it are a little deal  , it have no one  them for my wife ,  do not come home and  do , the paddock says  . the paddock says ,  no , i have not  for little deal which my  queen which a little deal  man , and the child  which the son which the  child which the daughter which  the son which the daughter  said , `` what are  not my wife . the  girl says to her which  not as long as it  have no father and no  father the child which the  daughter said the child ,  which not a little deal  of money . the child  also and all one the  queen which the son which  which the son and the  queen which the son which  one one and the little  bed which the son which  the son which the daughter  the son and the son  until it were all so  dead until it and not  be so so the child  that it was very very  handsome that it was not  not so so that the  father said , this ,  that have you not no  little glass a great deal  for a great deal which  a great deal which my  great deal which which my  silver and gold which no  one , which no one  and a little deal which  it the daughter which the  daughter said the child ,  which not not as long  so it as it as  not as it as that  it also the son and  a little deal which which  one one which the son  the daughter which it also  the son which the daughter  which the son which the  queen which the son and  all the man and one  day the child which it  was so so that it  were not so so that  she were so very long  and soon that the child  the son and the daughter  the son which the daughter  also the son and the  woman and one of her  life and the daughter which  that the father said ,  it is the child ,  until the child which the  daughter said the child ,  that do not know what  it is so long as  you have it . you  are not for my wife  the cry . the child  answered , `` what is  my child , have you  not my wife . the  paddock says , no ,  that have you not my  daughter . the paddock says  , no , no that  do you have my wife  . the girl which the  boy which the son which  the son and the son  which the daughter which the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 105)\n",
      "(39).(91).(7).(16).(87).(97).(59).(38).(9).(54).\n",
      "Average loss at step 106: 2.930018\n",
      "\tPerplexity at step 106: 18.727959\n",
      "\n",
      "Valid Perplexity: 84.19\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 105 ... \n",
      "======================== New text Segment ==========================\n",
      " never to be my wife .  that the king said ,  no , that not that  are my wife , and  that that the king was  full as long as the  king , and the old  woman said that the king  did not know how to  give the queen and the  old man said that the  queen would not be hold  and the old woman and  her daughter , and the  third that the king was  very great cry , the  woman said the whole night  that the king and not  all the king , whose  great joy which was so  beautiful that the king and  son and the king and  a great deal which that  that the king which the  king which had to be  all so half as that  the king , and the  whole palace which was so  so that that that the  old woman said that the  king would not leave it  . the king and not  how to leave out that  the king which the king  also and the daughter which  that the king lay an  king which the boy the  maiden which the son were  full of great joy ,  and that the king was  sitting for all the glass  an glass which which the  daughter was all for the  queen . the cat said  nothing that the king which  the king which was all  all for the glass and  not all as the king  with great joy which was  very long that the king  would not bring it the  cry , and that the  old woman said , ``  what are my dear wife  . the cat said that  that the king was so  very long that she were  not all so that the  old woman said that the  old woman said that the  old woman said that the  father said , `` and  my dear wife and not  be so so that the  old woman said that the  old woman said and the  king and not all a  long time and the king  the daughter and the king  with all great great great  wood which the king and  all that the king and  all that the king which  she was so long that  the king was so great  that that the king and  all a great deal which  that the king , whose  joy the king said ,  `` what are you my  daughter . '' and the  third said that the king  asked that the king said  which the king , son  , what no that no  dead , answered the king  to each other , and  give the queen and the  queen which the king and  great joy , however ,  should not leave it and  pieces and not not long  so that that was be  very so that the king  said , the king and  not be so long and  said , what are my  dear wife and be my  daughter . and the king  which the king and not   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 106)\n",
      "(3).(55).(91).(76).(96).(0).(4).(77).(52).(34).\n",
      "Average loss at step 107: 3.201803\n",
      "\tPerplexity at step 107: 24.576801\n",
      "\n",
      "Valid Perplexity: 80.35\n",
      "\n",
      "Generated Text after epoch 106 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " the bride and the king 's  daughter , and the king  's daughter and the king  , who and no one  and the king and all  a great deal one .  she went home and the  king and the old woman  and the old woman and  the old woman and the  old man and the king  with all speed the king  's son and all the  old woman said , i  come home and eat and  shape and eat it .  the king and all the  king and all all the  all all all all all  all all the all all  all the all all all  the great bed , and  cut her head and and  that the king and all  his daughter was so that  it should not be hold  until the house and no  great glass full and all  gold . the bride said  that the king said ,  how not no one .  my dear bride , and  let it go home and  be my wife . the  bride and the king and  his son and the king  , who and the king  , and the king and  the king , and the  king 's son and the  old woman and the old  man said that the king  would not know it and  it . the king and  the king and all the  queen which the bride which  that the bride was so  all that she and the  old woman and the king  and the old woman which  the king and all the  king and all all the  all all all all all  all her all and the  maiden which she the bride  the king , and put  a little bed and and  the king and all the  king and all all the  great bed , and went  to her little house and  which the king was full  one and her one and  his daughter , and the  old woman said , how  have you not my wife  to eat and eat my  way home and eat .  the king and all the  king and all all all  a great great glass which  it and all all all  all the all all all  all the all which the  boy which the daughter which  he had to return to  her wife and her wife  and the king and all  his daughter was so long  that the king and all  all all the great glass  one and the king and  the king and all the  king and all all all  her the king and the  king 's son and the  whole house which the king  said , how not you  may be my wife .  the king and the king  with all great great hole  for the king and all  a great glass which the  queen and the bride and  all it and all all  her one and the king  's daughter . the king  with all speed and had  hold one and all the  glass which the bride were   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 107)\n",
      "(16).(64).(6).(44).(67).(38).(45).(94).(1).(19).\n",
      "Average loss at step 108: 3.249558\n",
      "\tPerplexity at step 108: 25.778939\n",
      "\n",
      "Valid Perplexity: 75.70\n",
      "\n",
      "Generated Text after epoch 107 ... \n",
      "======================== New text Segment ==========================\n",
      " her wife , and went home  until the door , the  fox said that the king  would not believe the queen  . but the third one  of the queen were all  his daughter , and at  each other and said that  they had not no one  drink , the king and  all that she had not  all the queen which the  bride was so long that  they had not no one  power in the bed ,  she went home and went  home and went to her  little bed , and at  the last one of her  heart were so beautiful the  whole morning , the wolf  was so so that the  maiden which he had and  each one them and said  , i have no one  any one . '' the  youth went home and went  home and kissed her and  carried her and the bed  under the great bed which  began to cry and and  how so the old woman  said the queen , how  no one can be your  little and death that i  will not leave you .  '' the girl began to  return to the door ,  she went home and went  home and kissed her and  ate her and joy the  queen began to cry and  until the old woman said  she the queen were all  so beautiful that the old  king which the queen had  the queen , and the  maiden began to cry and  went to her little bed  , and at last the  old woman said that the  fox was not so long  as she said , the  child , i have no  answer . '' the girl  and the queen went home  again and said the queen  . i will not be  your wife , and that  you will not be your  wife , and that the  king and all a great  stone be full that the  little girl said , i  know that you are so  beautiful that i have no  human eat . '' the  youth said , `` dear  father , i have no  other money and your little  house . '' and went  into the door , and  said the king , and  now that i have not  a great glass which the  great glass which which the  king which the queen is  for the queen . the  old woman said that the  queen were not so long  , and had no one  drink to her little bed  , and the old woman  began to cry and the  maiden and the old woman  began to cry and the  maiden and the king and  that she had not not  leave her and said that  they had not no one  drink . the wolf said  the queen which that i  do not know what i  have . that that that  i have not no one  any one . '' the  queen went home and went  home and kissed her and  carried her and the bed  and the queen and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 108)\n",
      "(65).(92).(61).(29).(40).(56).(36).(42).(39).(97).\n",
      "Average loss at step 109: 3.140529\n",
      "\tPerplexity at step 109: 23.116097\n",
      "\n",
      "Valid Perplexity: 81.84\n",
      "\n",
      "Generated Text after epoch 108 ... \n",
      "======================== New text Segment ==========================\n",
      " a stone . the old witch  would have it no one  , the king and the  daughter , and the king  which the daughter and the  king and the daughter and  which the daughter , the  king , however , had  the king and the king  and the daughter , and  the king and the daughter  all the king , and  and the woman , however  of all the old man  of the old woman were  all delighted , and that  as the old witch were  full of joy , and  it and no one ,  that the king and the  king and the king ,  that that the king and  the king and the king  , the old woman were  all an king and the  daughter , and the king  which the daughter and the  king and it were so  white that the old woman  , however , were so  only that the old woman  that the old woman said  and the king , how  not no one to be  your wife , and the  king and the daughter ,  and the old woman were  all delighted , and that  as the old woman said  nothing for an king and  the daughter , and the  old woman were all delighted  so long that the old  man said , `` that  have not no one any  of it . the king  also at each other and  the daughter and the king  and the son , and  the cook , and the  king and the daughter ,  the king , how no  one ever have no one  money home home and drink  . the king and the  daughter and the king and  his daughter , and the  woman , however , were  very handsome , and the  old witch said , ``  and that you have no  wife , you are a  dragon . the old man  said that the king ,  what no that no one  ever have no one you  not my wife , and  what not no as the  cook , and the king  's daughter , and the  cook , and the cook  with joy , and the  woman and the king and  the daughter , and the  maiden , however , were  all still dead , the  king of them and the  son and the king and  the daughter , and the  woman , however , were  very handsome , the children  which the daughter and the  king and the daughter and  the king and the daughter  that the old woman were  all joy , but the  old woman said that the  king , how no one  can not be my wife  . the king and the  king , how no one  can not be my wife  . the king and the  king , how no one  can not be my wife  . the king and the  old woman were all delighted  as not as it as  that and the king and  his daughter , and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 109)\n",
      "(40).(99).(88).(46).(97).(24).(57).(61).(20).(83).\n",
      "Average loss at step 110: 2.958826\n",
      "\tPerplexity at step 110: 19.275320\n",
      "\n",
      "Valid Perplexity: 80.26\n",
      "\n",
      "Generated Text after epoch 109 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " rich that it would not be  hold for a little light  for the little bed and  it in it . the  man gave him a little  little light and its bed  , and began to sing  which it in the little  pieces which which the daughter  and the daughter which it  the daughter , and the  daughter which it in the  wood which the daughter which  began to sing that the  man came home and sing  that the little bed ,  until it hold for the  four bed , and when  it it at all the  night the child came and  rest and rest which it  and all a little bed  its little bed and which  his wife wife which it  at last the child came  home and laid it so  a bed and it it  so long that the child  also and the daughter which  the daughter the daughter which  at each other , the  king and the son and  his daughter , and it  began to stop that the  little brother came and and  that the king which were  son the daughter , and  the daughter and it at  this all which came in  a little bed which the  queen which the daughter which  the daughter the daughter which  the son which came for  her little bed which came  in the bed and the  whole light , and in  the bed which the daughter  which the son which were  laid in its bed and  laid the little light which  which came in the bed  so a little bed which  which came for it and  carry it a little bed  for the little bed and  the king and the king  's son which the son  which were so beautiful that  no little man which had  the daughter in its bed  or the bed , and  made a little light and  a little bed which which  himself in the bed ,  gave him a little bed  and which it in a  little bed and the king  home and drink that it  would not rest in the  little bed and which the  whole light , and as  he gave it all the  half which which it in  a little bed and the  whole light . the mother  gave her all a little  white bed which which came  so a little bed which  which began to sing that  his little bed , and  made a little bed and  his bed which the daughter  whom it had not no  use , it came and  wife it in the bed  its little bed which the  daughter which it in it  for a little bed which  the daughter which the daughter  and the daughter which it  at god in the bed  and heard the little light  . the king and all  all which which came for  a little bed and the  king , and the daughter  and the daughter and it  the daughter , and it  began to sing that it  came home and sing that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 110)\n",
      "(12).(59).(35).(36).(17).(86).(8).(21).(0).(18).\n",
      "Average loss at step 111: 3.048980\n",
      "\tPerplexity at step 111: 21.093828\n",
      "\n",
      "Valid Perplexity: 78.53\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 110 ... \n",
      "======================== New text Segment ==========================\n",
      " 's king , and the little  sister said the king ,  and the king 's daughter  , and the king ,  and the king , who  came and meet her and  which the king , who  the king , who was  for a little glass which  which the king which no  son , she was so  glass that the king and  the king and all which  she heard for the glass  of a little glass which  this that the king and  no human form . the  man and the king ,  which the king 's son  and the king and this  human form and the king  's son , and the  king which she the king  which the son heard that  she was no little glass  which it the king ,  and the king 's son  came home and went home  to the king , and  the king and all a  great glass which which the  young king which the king  which the son heard that  she was no little glass  that the king , and  that the king , and  that the little child came  for and the king ,  how the king which the  little son heard the little  girl , and the little  tailor , and the king  's son which the king  , and it was so  glass that the king and  the king and all which  she heard for the glass  which it was a little  ones in the glass which  the king which the son  saw that it the king  's son which the king  also which heard the little  brother which the king and  the king , and the  maiden , and the king  and this human form ,  until the little dwarf said  the king , and the  maiden , and the king  and this human form ,  until the little dwarf said  , what do you do  and be my wife .  until the little tailor came  and the king , and  she went to the king  , which the king which  it the king which she  heard that a little little  glass which the king which  was no one to be  your wife . but the  devil came home and went  to her and she went  to the little door and  said , what do you  want , but my father  what i do , and  be my wife . the  little tailor said , what  come home and my father  which you have for my  little glass which i not  go home , and not  the king 's daughter .  '' the king and the  king , and that the  maiden , and the king  's son and the king  also which the son which  heard that and the king  and the king and the  dragon , and the little  light which the king which  heard that . the little  girl said , what come  to my father . ''  until the little dwarf said  , what do you be   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 111)\n",
      "(87).(55).(4).(81).(38).(56).(49).(8).(0).(93).\n",
      "Average loss at step 112: 3.029393\n",
      "\tPerplexity at step 112: 20.684682\n",
      "\n",
      "Valid Perplexity: 80.79\n",
      "\n",
      "Generated Text after epoch 111 ... \n",
      "======================== New text Segment ==========================\n",
      " if it do not know what  the king , son which  heard that . the king  were so so so that  she were all so long  , and when the king  came and the king and  this that he were so  beautiful that they were so  son that the king and  his daughter , and the  old man , and the  young king 's son ,  and the king and this  long that they were all  laid so half so half  that that the king which  which the king which were  so beautiful , the sun  saw all his hair and  all a little deal which  it which the king and  the king which were all  full of glass his hair  with a great deal which  one which which had no  rest and the king ,  in the door , they  all a great deal which  that the king which he  he were so so that  the king which the son  were full and all a  wicked man and all a  great deal which which his  son his son , which  the king 's son and  the king which it which  at last the king said  he was no one ,  and the king 's son  were full which that the  young king 's son ,  and the king and this  great joy , and the  man , and it and  not know it , and  that the king and this  white which the king and  death , but the king  began to cry , but  the king said , ``  i do not know that  that the king and all  a great deal which that  his father which the son  were full of joy and  long as they were so  only that the king and  a great deal which one  a wicked man , and  that a great deal which  which the king which were  very beautiful and all a  great deal which that the  three wife which the king  began to cry , the  man and all a great  deal which it which the  queen , and the king  's son which the son  were so so that he  began to cry , how  this the king which he  were so beautiful that they  were all laid in the  bed . the king said  to the king , ``  now you do not be  as as the king and  the king 's son which  the king and this one  which the king which were  only full of joy and  long as they were so  so that he were so  long that they were so  son and that it and  all a great deal which  which the son which the  queen , and it and  not know that , but  the king said , ``  i do not know that  that your little wife which  that you do not know  how to do it .  '' the tailor said ,  now the father and this  if the king , that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 112)\n",
      "(50).(78).(60).(74).(64).(7).(1).(23).(9).(56).\n",
      "Average loss at step 113: 3.138626\n",
      "\tPerplexity at step 113: 23.072138\n",
      "\n",
      "Valid Perplexity: 78.87\n",
      "\n",
      "Generated Text after epoch 112 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " , but the king said ,  what do you do .  how do you do that  my dear wife not do  not have my dear wife  , and do n't do  what you do , and  have you be my wife  , and the fox said  and meet her all a  wicked man , and the  king and son and the  king and son and no  daughter , and that the  old woman which she had  so only that the queen  which only that the queen  had only not be so  so that the queen which  no one , which were  so only as a little  wife , and the queen  let him be full which  not the queen of her  daughter , and the fox  which the son which the  son and the queen and  his daughter , and the  maiden began to walk home  again , and at last  that at the third time  that she went and put  it in her bed and  cried to her and not  the king , and the  fox which the queen and  the son and the queen  began to walk home and  appeared that she was no  beautiful wife and how the  king and rest for a  wicked which which the queen  , and it went in  until the fox and the  queen home and bed her  for the queen , and  the fox cried to her  , which do not be  full which do not be  my wife , and the  king said , what have  you do and have my  dear wife and you do  n't do you , and  do you want to take  my child home . you  may not be my wife  until what do you do  and do you have no  rest . that is a  fox , how you have  no one do you do  for me , i have  my daughter , you have  my wife , how do  do do that that do  be my wife , and  the fox said , go  which the fox have not  not my wife . what  that do that you have  my dear wife that i  do not be my wife  and what do you do  . what do you do  that my dear wife wife  she must not be my  daughter , but that i  not do you and do  not want me to lie  where you do that that  you do not be my  little wife , and that  have you do not have  it for my wife ,  and do not be my  daughter , but that i  do not be my wife  until what do you do  . what do you do  , and do you want  that you do do you  i do not be my  father , and the third  girl and not be so  long , and that it  in the stable and the  old woman which it was  not so only for her  all such such such such   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 113)\n",
      "(68).(18).(63).(88).(96).(43).(48).(49).(53).(60).\n",
      "Average loss at step 114: 3.157414\n",
      "\tPerplexity at step 114: 23.509728\n",
      "\n",
      "Valid Perplexity: 79.29\n",
      "\n",
      "Generated Text after epoch 113 ... \n",
      "======================== New text Segment ==========================\n",
      " not the king , and the  man said , i have  nothing for me . the  little dwarf said , i  will not meet it .  and the man said ,  take me and that my  wife is not a fine  fine fellow my wife ,  or that you have no  wife . i have no  other than that that i  do not know that that  my wife are not a  great deal which you have  no wife , and the  king said , i have  the king , and the  king and that the king  and the father said ,  that is a fine deal  man . what is you  for me , you have  my daughter , and the  devil said , i have  nothing for me . the  king said , i have  not a great deal of  money and my wife ,  said the father , i  will not meet me .  but the man said ,  what that that no one  can not be my wife  for me . i have  a great deal of money  for me . the father  says , i have no  trouble for my father and  that my wife , will  not take it home .  the man said , i  will not meet it .  but the man said ,  now that no one shall  eat my knapsack and the  man answered , i have  it for my wife .  the man said , i  will not meet it ,  or i not not no  beautiful than the father ,  but the father said ,  no , i not not  meet it , and the  wedding was full which that  was so beautiful that the  man said , i have  nothing and that my wife  that i not not a  great deal of money and  my wife . the father  answered , i have no  father and you that no  father which the son is  my wife . the man  replied , the man is  my daughter , and that  my father , may the  horse you have no one  for him . the man  says , i have no  human money . i have  a great deal which you  you have it , and  not not have it ,  or that i are so  long that that that the  little man is not a  beautiful fellow and eat ,  but that my wife is  all me for me and  have a great deal which  bread . you must not  be my wife , and  the father said , i  can not know that i  not not no one ,  that that my father ,  i not not know that  that no one has a  glass deal and death ,  or that that i is  not my wife , and  and the father said ,  no , i have not  my daughter to eat .  that that is to meet  you , i have no  fear , you have no   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 114)\n",
      "(1).(82).(83).(67).(64).(38).(53).(70).(9).(58).\n",
      "Average loss at step 115: 2.832831\n",
      "\tPerplexity at step 115: 16.993501\n",
      "\n",
      "Valid Perplexity: 79.66\n",
      "\n",
      "Generated Text after epoch 114 ... \n",
      "======================== New text Segment ==========================\n",
      " UNK . the old man began  the cry , the man  began to eat and eat  to him and said ,  my dear child , and  go and eat it and  the king and death ,  but the old man was  home and said that the  father says , what are  no little wife , you  do not know that the  old man is home and  not not a little and  a little wife which no  man , and the wife  looked at each other for  a little bed and said  that the father said ,  my dear child , and  eat the glass which my  father , what not my  wife , have you not  be my wife . what  do not know what that  have no wife . what  are you dead for my  death and take my wife  my wife wife and the  little man and the wife  the son and the daughter  which the son began to  cry . the king ,  and the father and daughter  and the son and the  king and son and the  king , and the old  woman began to cry .  she at once and son  and the son and the  queen which the son which  began to cry . the  man and the old man  began to eat home and  eat the little home and  it for a great deal  of money and the king  . the man said and  only that the father were  full which that the wife  began to cry . the  man and the old man  carried her and the woman  again and the other and  began to cry . the  man and the old man  began to cry and eat  again and the queen and  the old man and the  man and the old woman  and that that the man  began to eat and eat  to him and said ,  my child , what do  my child , and not  do for the king ,  for the old woman had  the queen , and the  queen , and the father  and son which no one  , and the king ,  but the old man were  still dead and the man  , and the old man  also and the son which  had no wife to eat  him and the queen and  the wife which the son  , and the king and  son and the son which  at last the daughter and  his wife and him and  she began to cry .  it and the son ,  began to cry and cry  the cry , and the  king and the old man  which the son and a  little white flowers which which  which at last the daughter  began to cry . the  man and the old man  carried her and the maiden  again and the old man  began to eat and eat  . the man began to  cry . the man and  a little white white flowers  which which began to eat  home and said the child   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 115)\n",
      "(32).(55).(17).(34).(64).(73).(30).(98).(94).(23).\n",
      "Average loss at step 116: 3.098575\n",
      "\tPerplexity at step 116: 22.166331\n",
      "\n",
      "Valid Perplexity: 84.58\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 115 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " man to gretel . what do  what do you have .  what do you do .  hans said , hans home  . '' the girl said  to her , what have  it like be . but  what do you take that  for me . what have  the devil , you have  nothing . what do you  take . what do you  know . what do you  have . what do you  do . what do you  give me . what do  what do you have .  what do you do .  that do n't take it  to my wife . gretel  , hans . what do  n't what do do that  it is to be my  silver wife . what do  n't do what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  give you . i have  nothing but death . ''  the man said , ``  hans . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  know that . '' the  old man had her daughter  of her , the man  were all all dead .  and day , hans .  '' the girl said ,  what do you do .  you do n't know what  that do to be .  what do you do .  what do you know that  . '' and the third  said the third , ``  hans . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  know that . '' the  girl said , `` what  do you have . i  do not know what the  old man would not have  for her and do not  end . what do you  do . what do you  do . what do you  give you . i have  it for my wife ,  hans . what do you  do . what do you  do . what do you  know that . '' and  the third and the old  princess came to her and  which the old woman were  home and said , hans  , you do n't know  it . what do you  know that your wife have  the roast meat that a   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 116)\n",
      "(7).(4).(71).(82).(73).(5).(98).(90).(34).(19).\n",
      "Average loss at step 117: 3.279809\n",
      "\tPerplexity at step 117: 26.570688\n",
      "\n",
      "Valid Perplexity: 73.63\n",
      "\n",
      "Generated Text after epoch 116 ... \n",
      "======================== New text Segment ==========================\n",
      " , but the king said that  no one are you .  but that no one may  come home and see that  that is not my bride  . and that that the  young bride have him ,  she will not be too  long . but that the  king and the bride the  boy , and was so  beautiful and the king said  and went and said the  children , and went home  from the house and the  whole tree and the house  was so long that the  maiden and all the king  until the old woman was  once like the king and  how no one . the  man went home and went  on her eyes and went  out and went and the  window her and and the  maiden and the king and  how she had no one  money in her life and  she went home and went  home and went and the  window that , and went  on and said , ``  that that you are not  here than that my dear  wife , i have no  little life , i have  the bride be my wife  . but that no one  may not be my wife  . and the father said  to her , i have  the bride be my wife  to be . '' the  father said , `` what  do you do . ''  and the king went home  with her and went and  cut the great bed and  opened the door , and  now that she had no  son to pass that the  man began to cry and  that the father and the  daughter and the king and  how she had no one  money in her life .  she went home and went  on the house and the  door which the son and  no one , and the  king , and the king  and how no one .  how the king said the  bride , you i have  it to be my wife  , but that that that  no one may be my  wife . and the father  went home and went home  , and no one ever  that him and see that  his father said , what  no one may be my  wife . but the king  and not how he had  not no one any one  . the king and not  answer and had no one  sleep on the basket and  went home and went home  again and said , ``  and not do that that  do you be here ,  until that you are no  like that that i not  be here . the king  said , `` and no  man , you have no  one you that it and  do not know what you  not . i have no  rest for them . the  father said , `` what  no that i have the  bride , you that you  will be my bride and  see that i have no  wife you not be my  wife . and that the  king , however , had   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 117)\n",
      "(20).(42).(45).(75).(81).(6).(21).(78).(32).(17).\n",
      "Average loss at step 118: 3.264415\n",
      "\tPerplexity at step 118: 26.164791\n",
      "\n",
      "Valid Perplexity: 77.03\n",
      "\n",
      "Generated Text after epoch 117 ... \n",
      "======================== New text Segment ==========================\n",
      " said she , i have no  one be my wife .  but the old woman said  , `` i have no  man who your wife have  no one be my wife  and that that that that  are you so long as  that that you that the  king and death , and  take her to my wife  , and said , ``  you are him that that  i do not know that  and do you for my  daughter . i have no  money so long as it  are so long that the  old woman said , i  know that that are you  so long for that that  i have it for you  for my wife . but  this man would not be  still like a man .  the girl , and the  king , and the old  one , and said ,  that do you come home  so long as you do  for my wife death .  the girl said , ``  that is not my wife  to be my bride ,  so that you have no  long long as you do  . the man said ,  this that is all my  dear wife , and said  the young girl , and  all a great stone which  so long that the old  woman were all covered and  bed . it was not  long as the old woman  saw that the king was  so beautiful and that she  came and sleep so long  as the old woman which  all as long as the  king said , i have  it be be for my  luck , i not not  be my wife . and  that this that the devil  would not come home .  that as the devil was  daughter so so that the  maiden which the daughter which  he the daughter , and  that the old woman was  very long so long as  the old woman was all  full of death . the  old man went home and  leave her so that she  would not know that the  young girl said , i  come home . you ,  but the cock and the  tree , and put it  for her little bed and  went home and if she  said the king , and  go home , and the  devil said , i have  the bride , and the  little girl said , ``  you are him that and  go home . you do  you for my wife ,  and that that that the  king is so long that  that that that no little  than death that i not  be my wife . but  it would not know that  as if she would not  have it . the girl  , and the old woman  came home and put it  home and put it so  as that the king was  a handsome man , and  how no long as the  old man would not be  too one . the man  went home and put it  in the bed and said  that the old woman would   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 118)\n",
      "(51).(18).(55).(88).(76).(9).(92).(43).(28).(23).\n",
      "Average loss at step 119: 2.824878\n",
      "\tPerplexity at step 119: 16.858893\n",
      "\n",
      "Valid Perplexity: 83.87\n",
      "\n",
      "Generated Text after epoch 118 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " daughter to her . what do  what do you do .  what do you give you  or death . the little  hare says , what do  n't do that that do  my wife be my wife  and gretel , hans .  what do you take the  axe and not not that  that no one . what  take you to that .  '' and the girl and  everything the queen and not  have to eat and drink  , hans , i not  take her to my wife  until gretel . gretel ,  wife , hans and the  little woman said , what  give you a little child  . what do you do  . what do you give  you . what do you  find that that do you  do . what do you  do . what do you  find . what do you  find that that do you  find . what do you  give me . what do  gretel . what do you  give it . what do  you do . what do  n't you give . what  do you do . what  bring you my wife .  hans , hans . what  good thing do you do  for gretel . gretel ,  what do you do .  you have nothing but you  , what do you do  . what do you do  for gretel . gretel ,  good evening , hans .  hans says the child ,  i have nothing and you  bring . what do you  do . what have you  do that i do not  know that , answered the  girl . the little dwarf  goes in , and the  king and death , and  the little woman said ,  i have nothing but take  me . what do you  do . what do you  do . what do you  find . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  find . what do you  do . what do you  do . what do you  do . what do you  find . what do you  bring . you have no  human form . what do  i give you . i  take it home . what  have you do my wife  and gretel , hans .  i do not know what  your little wife says ,  hans , hans . what  do you do . what  bring you my wife .  what do you do .  that do you do you  but , '' and the  queen and the little man  said , what no and  take my life . what  do you do . what  have you my wife .  you do not know that  the little girl is so  beautiful and that i would  take my wife and what  what do you do .  what do you do .  that no one would not  find it . what do  my child , hans ,  what do you do .   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 119)\n",
      "(50).(79).(46).(75).(64).(26).(33).(2).(19).(20).\n",
      "Average loss at step 120: 3.293562\n",
      "\tPerplexity at step 120: 26.938645\n",
      "\n",
      "Valid Perplexity: 75.31\n",
      "\n",
      "Generated Text after epoch 119 ... \n",
      "======================== New text Segment ==========================\n",
      " when the king said the young  king not not the king  and the king , and  that the king , however  which had no one rest  so long that the king  began to sing which the  man began to sing which  he were so beautiful until  he cut a little bed  until the man , and  it began to sing which  it was dead . the  young man said , i  do not know what the  little brother , said the  father , i have no  fear , you have no  so that that that the  wolf are not . what  not no as that the  little man said the young  bride , what no one  can be my wife to  eat and drink , that  it is so beautiful that  my little sister , i  see that you have no  like that that i have  it no one ever have  you for my father and  that my little sister ,  have you not my wife  , but that i have  my daughter in my wife  . but that is a  handsome man , and the  old man said that the  king said , i have  no one ever my wife  me , i have no  money for my wife ,  so that the king said  that the king said the  daughter , the king .  `` that that i have  my wife my wife ,  or that the king and  no one . what have  my father be my wife  , and that that the  fox are not so beautiful  for the little home and  which no little coat and  and the king , and  that the king , however  that was so beautiful and  said , what no one  may be my wife to  eat , what that have  no little wife , but  it shall not die ,  i have it no one  anything . '' the man  said the king , what  do you do . the  youth said that the king  , who came to the  king and and the king  began to sing which the  man began to sing which  he were so beautiful until  his child the young man  said , i have no  wife . you have not  for my wife the king  also , said the king  to eat , you have  not no one . you  will not take it home  , and not my wife  any one to be my  bride . the king said  that the king were a  handsome man , and the  king and that the man  came to light . the  king and that the man  came to light . the  little tailor and the king  and the king and the  young man were full and  bed them and until it  as no long long as  the old man were so  long that the wolf which  it in a bed and  which the king , and  a little man , and  began to sing which the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 120)\n",
      "(72).(40).(37).(38).(86).(9).(84).(67).(95).(11).\n",
      "Average loss at step 121: 3.202779\n",
      "\tPerplexity at step 121: 24.600795\n",
      "\n",
      "Valid Perplexity: 71.85\n",
      "\n",
      "Generated Text after epoch 120 ... \n",
      "======================== New text Segment ==========================\n",
      " who said the king , ``  do not get out of  you . '' and the  third said the king ,  what do you have .  but that is a little  man , i have no  wife . the little tailor  answered , `` i have  the king , and the  old woman said that they  had not no one any  like the bread . the  old woman said that the  father is a duck ,  the whole forest , and  how the king , and  that the little tailor ,  said the king , and  will not be my wife  . and the king said  the king , however ,  can not get on .  '' the king said ,  `` dear dear dear little  little dear dear dear dear  father , `` i do  help me and do not  do you . the king  , however , had no  answer and said , ``  my father , be a  duck . the king is  a great stone which which  no little little meat which  that no one , and  no one , and the  princess , and the king  and the queen , and  the king , however ,  would not help him to  do that the king and  it . the king ,  and the king , however  which had a king of  bread and the little forest  said , i do not  help you , and that  my little little tailor ,  said the king , and  i will not be a  white duck , and that  my wife , and the  little tailor said the little  princess , and that the  little tailor , said the  old woman , you have  a great stone which which  a little man and the  princess , however , had  an end which which the  princess , and the king  and the king which he  seized them on and and  his daughter , however ,  should not help him to  pass that the king which  the queen , and the  princess , however , had  the king , and the  king , however , had  an king , and the  old woman said that the  father is a duck ,  , i not not long  that , '' said the  princess , however , had  an king , and the  old woman said the king  . the little tailor which  that the king , who  did not help him to  get out , and the  old woman said that the  old woman had not be  a white man , and  that the king , however  and had a little white  horse which and the king  , and the king ,  and the king , and  the king , and the  princess , and the king  also , however , had  an king , and the  old woman said that the  king , however , will  not be my wife and  i will not help you  so long that you do   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 121)\n",
      "(84).(61).(69).(89).(16).(27).(82).(12).(43).(63).\n",
      "Average loss at step 122: 3.145405\n",
      "\tPerplexity at step 122: 23.229080\n",
      "\n",
      "Valid Perplexity: 76.16\n",
      "\n",
      "Generated Text after epoch 121 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " apple . the man said ,  no , i have no  father and death , and  that the little man said  , i will not be  your wife . and the  king said , i will  see what i have to  see that my name is  dead . and that the  little man said , i  must not have not no  long that that do you  be my wife . the  little man said , i  know that that is the  duck . the man said  , i must not have  my daughter . i have  my daughter to be my  husband . the man said  the father , i will  not be my bride ,  i will not be my  wife . that that i  will not be my wife  again . the man said  and that the little man  were in , and the  father and the daughter said  , i must go home  and you my wife ,  that i will not be  my bride , i have  my wife . the man  said , i will not  be my bride , i  will not be my bride  , i will not be  your wife . and the  man said , i will  see what i have to  be my wife , and  if you have not no  answer , and said ,  that is not my wife  . the man said ,  that is not my wife  , and that my father  said , i will not  be my bride , i  can not find my wife  wife . that is not  your wife . and the  little man said , i  must not have not no  long that that i not  see my wife , and  that the little man said  he was , that is  not my wife . the  king and not no one  in his life and the  old woman said , i  will not be my bride  and will not be my  wife . the man said  the father , i will  not be my bride ,  what is my name and  if you not not that  that you have no wife  for my wife . the  little man said , i  must not have not no  answer and said , i  must not know what that  is so long , and  as the little man said  he would not be my  bride , and the father  the daughter , and the  young man said , i  must not know what that  is so long , and  and that the little man  says , i will not  go home again home and  cut my life so that  you do not know what  i will not have the  riddle . the man said  and that the little man  said that the father said  the little man , and  she went home and cut  it so a great pieces  the little man said ,  my father , what do  what not my wife .   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 122)\n",
      "(47).(98).(58).(70).(45).(22).(86).(93).(79).(8).\n",
      "Average loss at step 123: 3.075334\n",
      "\tPerplexity at step 123: 21.657119\n",
      "\n",
      "Valid Perplexity: 77.89\n",
      "\n",
      "Generated Text after epoch 122 ... \n",
      "======================== New text Segment ==========================\n",
      " out that the king which he  had so long until it  were all so so that  the child which the son  and the king , and  his son began to weep  until the king , who  came to the end which  it was so long that  the king which she were  all so long that it  came so long that the  king which the princess which  the son and the son  began to cry and the  queen , and the princess  came home and cut her  pieces and the king ,  however , will not see  you and do not see  my wife , and the  fox said , i have  my wife , and the  old man which the child  and a little deal which  that which the child which  the son which he had  who the king and son  ate and cry and it  as if the king which  was so long that the  old man came and let  her eat home . the  king which the son which  came so long that it  were all to drive and  and the old man ran  home and cut it her  until it as that the  old man came to the  place where the child which  came and let her see  it , and thus the  woman began to cry .  that the king which lay  for the king and the  bear which the king and  queen , and at length  the son and the old  maiden came home and laid  him in its bed and  and the king , who  began to eat and the  old woman which she was  so beautiful until the bear  . the king , however  the son which the son  and the king , who  fell in his sleep ,  but as it lay so  dead so long that it  that the old man which  were so long as it  that the king which he  did not see that it  the queen was not so  long that it lay so  dead until that the king  and the king , who  fell in his sleep and  let him be so long  , and as it lay  there so long that it  were so long as it  it the son and the  old woman came home and  cut a little stone and  it so long that it  was so long that it  was so so that that  one one which the son  and the king , who  came to the end which  it was so long that  the king and son and  the old woman came and  let him be so long  so that the old man  came and let him eat  out of the house .  the tailor said , ``  what do you see you  , '' and the bear  said , the bear which  a little child which have  not for your wife ,  you have not so long  that you have not so  beautiful that i do not  be my wife , and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 123)\n",
      "(81).(82).(61).(32).(3).(74).(1).(87).(60).(0).\n",
      "Average loss at step 124: 3.176634\n",
      "\tPerplexity at step 124: 23.965940\n",
      "\n",
      "Valid Perplexity: 74.13\n",
      "\n",
      "Generated Text after epoch 123 ... \n",
      "======================== New text Segment ==========================\n",
      " a great deal which death which  death and the king 's  daughter and the king which  he had in a bed  or the king in heaven  or i do not know  it , but the king  answered , i must not  know how to be my  father , and that you  do not know that you  can not be my most  long one , and you  must not be my king  . the tailor said ,  go home and cry and  not that , '' and  the king and him and  said that the king was  the young man and the  queen 's son had no  son , and the king  's daughter and the king  and the king 's son  again and the king and  his daughter , and the  sun had not no one  money to light . the  tailor said , `` that  no one will be your  wife . '' and that  this that no one would  come and meet it and  sleep which the king 's  daughter , and the king  , and the king ,  how no one can be  my king . the king  and this all a little  white man which the king  's daughter and all which  which had no one any  for any little which the  queen said the king and  cry . `` i have  it . '' `` that  that that i not not  as you . you are  the king and death ,  until the king and the  old king was so beautiful  that it which no one  would not be so so  a long long long long  that the king which no  wife and him a little  table , and the king  came and said that it  that no one would not  see that the king said  that i will not be  so long as you come  . and that you are  my dear daughter , and  that you do not know  that you will not be  so so that the king  replied , you have you  it . '' the tailor  and the king , however  the daughter which the king  came home and kissed her  that the king which was  the king , and the  old king 's daughter ,  but that the king was  a little deal which which  which the king which no  great joy which she had  come and said , ``  you will not be my  daughter . '' and when  this man were all behind  her little bed and said  that the sun were so  long as that a little  bird said the king ,  i will not be my  wife , and that you  do not know that you  do not know it ,  or you will not be  no one for me .  what do you do and  stone or you that i  want to light . ''  the tailor said , ``  i will not be my  son , '' and the  king and the king and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 124)\n",
      "(15).(97).(73).(61).(90).(66).(56).(5).(28).(32).\n",
      "Average loss at step 125: 3.065779\n",
      "\tPerplexity at step 125: 21.451174\n",
      "\n",
      "Valid Perplexity: 78.09\n",
      "\n",
      "Generated Text after epoch 124 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " his daughter , and the king  and death , and the  king , and the king  's daughter , and the  woman said , i have  it for you , i  do that that you have  it for my wife ,  you have a little ones  home and drink that you  may be there . the  king said , i have  no one for a little  fire which i have it  to carry it for my  wife and my little wife  be my wife and you  go home and carry it  out . it is not  long as that that the  old man , and the  jew which will not an  old man and the king  's son , and that  the king which is the  old man , and how  no one ever have an  little ones . the king  's daughter , that the  jew which said that the  fox is the king and  carry the little light which  the little ones belong to  old man , the king  's daughter , and the  man said , i have  the king which you have  the king and death ,  but that i have the  king home and be a  one , that i will  take my life and the  little ones belong to death  . the king said ,  no that i do it  , i do you that  you do not know that  that you do it .  what have you not my  end which you can be  for carry it for my  little wife and the little  light my little wife which  the little ones belong to  children , you have an  little ones . the little  man said , i have  it for you , that  do you do for me  . i have it for  your wife that it do  that that i do not  long as i do ,  but that i do not  long as i do that  i do that that i  have the little ones which  the wife which i are  in an work which it  . the king , said  the king , and not  no one belong to death  or you that i do  that that i have the  king home and be my  little wife and the little  ones home and be my  man and death , but  the little ones belong to  laugh that that i have  it for my life ,  do you do it for  this that you have an  end which which which is  the little ones home and  eat and drink that i  do not long that that  it , do you a  little ones home and drink  that i do not long  for that that you have  it for you , i  want the little ones home  home home and drink ,  if you have it for  me , i have it  for my life , do  you do it for you  . that do not end  . the king said ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 125)\n",
      "(79).(88).(96).(98).(37).(97).(82).(14).(83).(22).\n",
      "Average loss at step 126: 2.996517\n",
      "\tPerplexity at step 126: 20.015693\n",
      "\n",
      "Valid Perplexity: 73.50\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 125 ... \n",
      "======================== New text Segment ==========================\n",
      " some that the king and the  wife had her wife and  her wife and the little  tree which the king ,  that the king , however  . she had no wife  to be my wife .  the king and not that  he had no wife .  it was not that that  the queen were to be  all in the bed which  which had no wife and  sleep so long as it  was no one , and  made her wife , and  he went and said ,  i have no little wife  . my father and that  it is not my wife  for my wife and my  wife which my child ,  i have no wife .  my little tailor , and  not a little light and  my wife which my wife  and my wife and my  wife and my wife and  my wife which my wife  which your wife which my  little wife which my wife  which your wife which my  daughter , or my child  , that that my wife  have no wife and be  my wife . '' the  father said , `` what  is my child , i  venture it and my child  , '' and that it  is not that , i  shall not be my wife  . '' the child said  that the king came and  pass the bed which the  little tailor , and the  son said , `` lord  man , my child which  are my wife , and  no little wife and the  queen and the king which  the daughter came to the  table , and in its  great bed which the son  came and said , ``  have my dear wife and  god and not have the  child , '' and the  child said , `` dear  mother , that is not  a little man and my  child , '' and the  child said , `` dear  father , dear dear wife  and my wife and my  wife and my wife which  my wife which my wife  my wife and my little  light and my wife and  my wife which my wife  my wife and my wife  or my wife and my  wife and my wife and  my wife which my wife  and a little stone which  your wife have an wife  and the little tailor ,  or i can not have  it for my wife .  i have no wife .  hunger , '' and the  son said , `` dear  child , that is a  man and my wife which  my wife which my wife  my wife and my little  bed and my wife ,  and which you have no  one for my wife and  my wife , and and  you have no wife .  i do not know that  your wife have no wife  . '' the mother said  to the queen , ``  i am my wife and  it do that my child  and have no wife .  i have no wife ,  '' and the king and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 126)\n",
      "(99).(44).(1).(76).(32).(56).(86).(57).(64).(23).\n",
      "Average loss at step 127: 2.782739\n",
      "\tPerplexity at step 127: 16.163234\n",
      "\n",
      "Valid Perplexity: 80.87\n",
      "\n",
      "Generated Text after epoch 126 ... \n",
      "======================== New text Segment ==========================\n",
      " a new one . hans said  , hans , i have  it . i have no  father . what do you  not that . i have  it . you have no  one do you do it  or do . what do  what do you do .  what do you do .  '' and that she was  to be an end of  joy , and went home  with her . hans the  dragon said , i do  see you . hans the  queen '' the bear and  the king , who were  no one any one for  them , but the king  came home and drink ,  all was no one them  or death . hans the  young bride '' said that  for the king , hans  , i do not see  it . gretel do not  know that , '' and  it was nothing but her  . the mother , however  the dear child which she  was to be an end  which had an end which  it in an end which  which an end which the  king was so beautiful and  death that the little tailor  , however , did the  princess , which were all  all in such a great  bed . she went home  until the king which had  come to light . hans  hans asked her that the  king 's son and the  bride said , i have  nothing be so long that  she said , what do  what do you do .  or do you do that  hans do not do it  . what do you do  for gretel . gretel ,  what do you do .  what do you do .  what do you do .  what do you do .  that do you do for  death . i have no  money . hans , hans  to church , and the  old woman went home and  cry and the little ones  in the forest , and  it was so beautiful that  no one would be an  bride and the king and  the bride have the bride  which you have not a  end . i must not  see it . gretel do  you want . what do  you do . what do  what do you do .  what do you do .  what do you do .  what do you do .  '' and that she was  to be an end of  four white which which had  fall for death . the  bride , however , will  not be my wife and  her life . gretel said  it no that i do  it . i have no  father . what do you  see that you do not  want it . hans the  blue light . hans the  dragon was so beautiful that  no one should be an  bride and the bride which  had come home . gretel  and do it . gretel  do not believe that the  old king , and the  king and gretel . what  do you give you .  hans said , hans .   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 127)\n",
      "(5).(61).(92).(53).(55).(40).(0).(58).(93).(46).\n",
      "Average loss at step 128: 2.962479\n",
      "\tPerplexity at step 128: 19.345873\n",
      "\n",
      "Valid Perplexity: 72.46\n",
      "\n",
      "Generated Text after epoch 127 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " will do you that , and  that the king and the  queen and the king and  no one , and the  old woman , and the  queen and the king ,  and the king and the  young king and the king  's son and the king  began to cry , and  the young king said ,  i will not eat and  lie so that my little  little ones , and the  young king and the king  , and the king and  no one , and the  king , and the king  , and the king ,  said the king , and  the king and the king  's daughter , and the  old woman which the king  and the king and the  king and the king and  no one , and the  old woman , and the  old woman and the king  and the king and the  old woman which the king  and the king and the  king and the king and  his daughter , and the  old woman which the king  , and the king ,  until the king and the  son said , `` father  that i do not know  it and do the old  king , and now not  be my wife , and  it is not so fine  if my father , and  the king and the king  's daughter , and the  old woman which the king  and the king and the  son and the king and  the king and the king  also the king and the  old woman which the young  king and the king and  the young king , and  that the king and the  young king said , i  will not eat and drink  that you do not be  my wife , and you  will not be my wife  . and the king and  the king and the king  and the king and the  son and the king and  his son and the king  and the king and the  queen and the king and  it were so so that  she also the daughter said  the young king , what  do you do . the  man said , i will  be my wife , and  you do not know what  i do , and you  do not know what the  old man , and the  old woman and the king  , and the king and  death . the bear said  the young king , what  have you not my wife  and the king and the  queen and the king and  his wife said the young  old woman , i have  it . '' and when  they were so so so  as that the king which  it had no wife the  queen , and the king  and the king and the  old woman came and the  table , and the king  which the king and the  young king and the king  's son and the king  and the king and the  son said , `` father  that i do not eat  or not that you are   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 128)\n",
      "(15).(4).(33).(12).(36).(5).(63).(44).(96).(55).\n",
      "Average loss at step 129: 3.140172\n",
      "\tPerplexity at step 129: 23.107837\n",
      "\n",
      "Valid Perplexity: 73.41\n",
      "\n",
      "Generated Text after epoch 128 ... \n",
      "======================== New text Segment ==========================\n",
      " was the king , and the  king , who were so  only as that as the  old man , and the  king's bride which he said  to it , i will  not be my wife ,  and that that that the  little bride is so long  that the king which the  fox have come and take  her life and eat it  for a great deal which  it should be be so  beautiful and the king and  that that the king which  had the bride be an  maiden and the king ,  until the king , and  the king 's son and  an king and she had  a handsome witch , and  the king , however ,  could not get home and  asked it , and the  king said , i will  not be my bride ,  until you do not know  how to be my wife  , and that this that  said , i will not  be my bride , and  tell you that that you  will be my bride and  eat and drink , and  not a little coat for  which do not be an  king , and you have  my wife . what do  what do you do ,  that you will not be  my bride , and you  will be an bride and  the king , and will  not be an bride and  have the king , and  the king , and the  king 's son , and  the king and this all  joy the queen was not  long as the king which  she had to be so  beautiful that the king was  so great the old woman  that the king and the  king and the king ,  until the king , and  the king , who had  the king , and the  old woman which the king  's son , and the  old woman was so beautiful  and the king and the  king , who were so  beautiful that the king was  not so beautiful that the  maiden said , i know  it , do n't be  an king , and you  know that that do you  do for this that will  be my wife and that  i will not be my  son , i will not  go home . the king  was not so long that  it was not so so  that the king was a  great lake . that the  old woman was full which  the king and the king  and the king , and  the king 's son ,  until the king and the  king , and that the  old witch had not be  an bride , and the  king and the king and  the bride , and the  fox and the king which  one which which had no  wife and the king ,  until the king , and  his daughter , and the  old king to an end  of this which which had  an bride , and the  wolf , and the king  's son and the king  said , i will not   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 129)\n",
      "(54).(75).(17).(9).(70).(73).(10).(36).(96).(63).\n",
      "Average loss at step 130: 2.985430\n",
      "\tPerplexity at step 130: 19.795006\n",
      "\n",
      "Valid Perplexity: 75.18\n",
      "\n",
      "Generated Text after epoch 129 ... \n",
      "======================== New text Segment ==========================\n",
      " , and the king and how  the king , said that  the king , who is  all a great lake .  how do you do not  hear my wife , and  how you not no one  to be my wife and  that you do not know  how the bride do not  know that , and that  you do not know that  she would not be my  son , you are my  bride . the bride said  to him , i will  go and pass the forest  , and not not my  father . '' the girl  rejoiced the woman and how  the king , who was  not so long that the  girl which had not come  out and sleep and the  great forest , and the  father said , i will  not be my bride ,  but that is a great  noise . the man and  that at the third time  it the queen , and  that at the third time  she began to weep and  until the old woman had  his wife which the old  witch was so beautiful and  that that said that that  it is not so beautiful  for my wife and my  little dog . the bride  said , i will not  think my wife , and  you will not be my  true bride , and you  will not be my bride  which i do not know  that , and the bride  and the bride , and  that the bride said ,  that that will not be  a bride , and you  will not be my bride  that you do not know  you , and that you  are a great lake .  you do not not that  my father , '' answered  she , i are not  so beautiful that i do  n't know you . the  bride said , i will  be my bride , and  will not be my bride  . the bride , however  . she would not be  an bride , and that  the king , however ,  had a great lake .  if the father said ,  how do you not my  bride , and you are  the bride , and you  will not be my bride  that i do not know  how to be my wife  for my wife . and  she said , i will  take my wife home and  see that you do not  hear you . you do  n't know you . the  king , however , was  no one to be a  little stone and the little  boar which had the bride  for a great stone and  that the king , however  , had the bride come  and be a great stone  . the wild boar the  king gave him a great  noise and the queen said  it , and not my  bride , and how not  my wife . what do  you do . and the  girl said , i will  not be my bride ,  but that are all a  great lake . you do   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 130)\n",
      "(69).(23).(44).(73).(99).(86).(13).(52).(87).(79).\n",
      "Average loss at step 131: 2.709166\n",
      "\tPerplexity at step 131: 15.016749\n",
      "\n",
      "Valid Perplexity: 76.88\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 130 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " , but the jew said the  child , that is the  bride for the blue sea  . the jew says ,  no , that are you  a little tailor . the  paddock says , that is  the bride which do not  have my bride , and  you have it to be  your wife , and that  it is not all three  bit , you have it  so so long as it  like a bride , and  if you do not know  what the axe are so  death that it that it  is not so long that  he are so long as  no one . the girl  says to her , you  must be my wife to  life , and have a  little trade ? you have  the bride , and not  not a new one .  the girl did not know  how to be my wife  . the girl had no  one ever have it for  her little heart , the  paddock , and a little  wife which the daughter had  no one to bring it  to light . the paddock  says , no , you  are it for it .  this the child says ,  do you have for it  for them . the paddock  cried , no , you  do not know that that  you have no one for  her little heart , the  jew would not have not  long for it . the  paddock says , no ,  no that not that ,  she is a tailor ,  but that is a tailor  . the girl said ,  what do you have .  what do you see that  you are so long as  no one . the child  comes home and see that  the child are not dead  . the paddock says ,  no , you have the  bride have it . the  paddock says , no ,  you are the child that  that you have not so  fine long long as it  that it , not that  you are a fine fine  fine things '' and the  young bride said that the  bride would not know that  . the paddock says ,  no , you are the  bride and do not know  that , said the paddock  that the father says ,  no , you are the  bride and have the jew  . you are not so  long as it that it  like a bride . the  girl , however , kissed  the other one which it  are so long that it  did not know it ,  it brought him so a  long day and its dress  , and at last the  paddock had it in the  bed that it , and  his daughter , and the  little tailor which had no  one any one for it  . the paddock says ,  what that that that no  one are you . have  it my bride ? ''  this is the child and  be my wife . ''  said the girl , the  sun says to the king   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 131)\n",
      "(39).(22).(52).(85).(4).(2).(45).(48).(61).(97).\n",
      "Average loss at step 132: 3.285805\n",
      "\tPerplexity at step 132: 26.730480\n",
      "\n",
      "Valid Perplexity: 76.96\n",
      "\n",
      "Generated Text after epoch 131 ... \n",
      "======================== New text Segment ==========================\n",
      " upon him , and put her  for them so long as  the king , and the  princess , and the maiden  with the king which the  king which had an daughter  and the king , and  the maiden , and the  king , and the maiden  . the king and the  old woman went home and  leave her for him .  `` that that you can  be my wife , and  said , you shall not  be my bride , and  do not know what that  is so beautiful as you  shall have it . the  next morning the king said  to the maiden , ``  if you have not an  bride of you , i  are my bride , and  must not be my most  one of you have an  golden ring and do a  golden stone which your wife  shall be my wife .  until that a great stone  cried , you shall be  your bride , and how  the king , and the  princess , and the king  's son and the king  's daughter , and the  old woman , and the  old man , and the  maiden said , `` that  do not be like my  wife , and have you  my wife , and that  i will not be my  daughter , you have an  beautiful maiden , and my  bride , and have the  ring , and that my  father have a beautiful man  with my wife , and  it will not be my  wife , and have no  use so long as you  can be my wife .  but the fox said ,  how that you can have  an bride , and the  king and the king and  all of the king ,  how have you not my  daughter . you shall not  be my bride , and  will not be so so  so that it is all  that you shall be my  bride , and is a  strong stone and you have  my wife , and the  fox and the king which  all which the daughter which  the son was once more  until the maiden was once  like the king , and  put the ring which the  king and the son and  his daughter , and the  old woman , and the  old woman , and the  old woman , and the  old man , and the  maiden said , `` that  do not be like my  bride , and have a  great stone and it is  my bride . the man  also and the king ,  and the maiden , and  it in the sea and  be a stone , and  the maiden and each other  so the king said ,  i will be my most  wife , and have no  wife for my wife ,  but you shall be my  bride , and will not  give it to be my  father , and have not  my daughter . you shall  be my bride , and  is a strong stone ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 132)\n",
      "(69).(63).(97).(84).(18).(88).(58).(51).(14).(9).\n",
      "Average loss at step 133: 2.863368\n",
      "\tPerplexity at step 133: 17.520442\n",
      "\n",
      "Valid Perplexity: 77.67\n",
      "\n",
      "Generated Text after epoch 132 ... \n",
      "======================== New text Segment ==========================\n",
      " the little tailor , and the  third said that the king  had no wife , the  king said , what may  it take and what it  shall be my bride ,  until the little hare says  that that it are no  little wife , you are  the bride , and it  was a wife , and  the king said , what  can you be so fine  as i not . the  king said nothing that the  old woman had the daughter  until the little tailor ,  however , she had no  wife , it went and  pass and bed and the  little tailor , who was  nothing but long that the  man was so beautiful that  she should take her and  said , that are you  like my bride , or  not that it is not  an bride , and the  third the king , who  were no one more until  her little glass the little  tailor , who had no  human eat . the king  began to cry and take  it home . the little  tailor , however , was  a old woman , and  when the little tailor was  so beautiful that it was  no one to be an  bride , and the king  's daughter , and the  old woman had the wife  whom the maiden , and  his daughter to the end  , the old woman went  to her little bed and  laid her over her great  happiness and the little tailor  heard that the little tailor  of the wife , it  did not venture to cry  until the king , and  that the king was so  long as the little tailor  , and the king was  over , and the king  was once more until the  king , who began to  eat and drink , and  said , what do you  have to be my bride  or not that that it  shall be my bride ,  but the king says ,  take my life and take  it home and take my  life and what do you  do . and the little  girl said that the king  says , take my little  stone and take her life  or eat . the king  said , what may the  old tailor says , what  i have no wife .  the king says , what  no little thing i are  dead . the man said  , that are you like  your wife , and do  not know what it are  for the wife , the  king said , what are  it here . the little  tailor said , what are  it dead . and the  man said , what may  no one are my bride  also more than that that  it is no wife ,  the king says , what  no little thing i am  for the king , and  not not that the king  is so beautiful that i  have the king who are  still like a most man  , and will not be  here , or that you   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 133)\n",
      "(31).(24).(11).(17).(64).(47).(89).(88).(78).(15).\n",
      "Average loss at step 134: 2.997322\n",
      "\tPerplexity at step 134: 20.031813\n",
      "\n",
      "Valid Perplexity: 78.98\n",
      "\n",
      "Generated Text after epoch 133 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " the man and the bride ,  but the two brothers went  out and lighted the king  which it at all her  rest and the shoe ,  and at last the child  had rest which the two  other sisters , and at  last the child had rest  full which the son and  his son and the son  's son went home and  kissed her and went and  seek the wife and the  son at last the child  had rest which it at  an end , it had  an bride and the son  , and it had no  wife , it had no  wife , it had no  wife , the two woman  came home and threw her  down and the king and  she went home and put  his horse home and went  to the king and had  the shoe and the shoe  came to her wife which  fell at each other ,  but the two brothers went  to her father , and  went and put her foot  home and put her hand  in his bed and went  home and put her foot  into the bed and went  and put her arm and  the king and had not  no one to be an  rest and the shoe ,  he had no wife the  maiden went home and put  him on the shoe which  came on the bed and  went home and put her  into the bed and cried  , that that no one  are the bride and be  an wife and the son  's son will not be  too so as that for  me . you are no  one more one for them  , and you have no  fear , you have no  one for it . the  man 's son and the  daughter and the son went  to the shoe and said  that the king , however  . the king had not  the king in the shoe  . the peasant 's son  that the king and the  two sisters which she had  not no one any one  to be an bride and  it , however , had  the king 's son and  his son and it was  a son and the shoe  and the son had the  shoe him her shoe which  it had so so that  that the two sisters had  rest for the two and  man 's son and the  bride and the son ,  until the king at the  right other which had who  the king and the son  did not know what the  king 's son had the  wife 's son and the  king and had no one  to be an bride and  it , who had no  little son and the son  went home and put her  on the shoe and cried  that the king and the  son had the bride be  full of gold . the  king at last the child  came home and went home  with her , and the  king 's son went home  . the king 's son  the son went home and   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 134)\n",
      "(10).(33).(95).(73).(70).(7).(57).(24).(97).(11).\n",
      "Average loss at step 135: 3.016790\n",
      "\tPerplexity at step 135: 20.425624\n",
      "\n",
      "Valid Perplexity: 77.22\n",
      "\n",
      "Generated Text after epoch 134 ... \n",
      "======================== New text Segment ==========================\n",
      " money home , but the king  did not know what it  is so beautiful that you  shall not be my wife  , and not not that  my dear wife , and  if you do not know  how my father is so  small than that the king  , and the king ,  where the king , however  . they would not be  like so use , you  do not know what my  wife is so beautiful that  i will not be an  bride '' and the king  , and the king ,  and the king , who  did not know that the  wild beasts which would not  know that the king was  so beautiful that the king  had her rest which it  was so beautiful that the  old woman , however ,  would not know what the  queen had to be an  son and it must be  an bride , and the  king and the king 's  hut , and it did  n't know it , and  the king , however ,  did not know what it  would be an son ,  said the king , and  do not know what the  king , and the king  's daughter of them ,  the little tailor , however  . she would not be  too use , and the  wild beasts which had no  rest and a great happiness  a great stone which which  is so beautiful that you  can not be my wife  , and not not that  my dear wife and not  not that you should be  set over her little little  sister . the man did  n't know it , and  the king and the king  , who did not know  it , and said that  they had not no one  any one pieces . the  king said , how do  my child , how do  you do my wife ,  which i will not not  see it , and the  king , said the king  of a great forest .  you will not be my  wife , and have not  no one to be my  father . the man said  to the king , how  do you do for my  father , i will not  be my wife , and  if you do not know  what my father , answered  it , the whole forest  . and the king ,  however , had to be  hold of a little stone  the little bed which had  no rest until the forest  . the man did not  find it . the man  did not know what the  king , and they did  not know what it should  have no one . the  man would not know what  that had to be as  long as the wild beasts  , and how it is  not no use , ''  '' said the king ,  dear father , i will  not be my wife ,  which will not be my  son , i have no  fear , and not be  my wife , and that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 135)\n",
      "(8).(63).(89).(75).(20).(26).(71).(14).(30).(36).\n",
      "Average loss at step 136: 2.963476\n",
      "\tPerplexity at step 136: 19.365174\n",
      "\n",
      "Valid Perplexity: 80.03\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 135 ... \n",
      "======================== New text Segment ==========================\n",
      " fresh length the wild beast ,  which went home and cut  his little dog and the  tailor and the tailor ,  when the wild boar ,  he went and cut him  its little dog and the  dog which had no wife  , the dog and the  cat . the dog and  all the cat , the  cat in his bed and  went out and cut a  bit and the dog and  each one exactly of the  dog and the cat ,  that was no one to  go out . the boar  went out and cut a  bit which the dog which  it was long as that  for the dog and the  wild boar had to be  like a handsome man ,  which knows what the wild  boar would come out and  drink the wild boar had  not rest for his wife  until the wild boar ,  said the boar , and  how not no one for  a wild boar , and  not not that that is  in the wild boar ,  which have no one for  your wife , and do  no one , and the  whole tailor . that the  young man would not be  dead . the dog and  ran out and crying and  that the wild boar were  dead and said , that  the little boar goes home  again . the wild boar  ran home and cut it  to the wild boar and  drink the wild boar ,  but the wild boar would  take him home and sing  that and said the wild  tailor , however , are  you dead . and that  that the wild woman were  full of water . the  boar ran out and cut  it and the wild boar  ran out and cut it  in the wild forest and  had not no one to  eat it . and the  tailor was so long that  a dog and the dog  which had rest and a  handsome man , they went  into his bed and went  and cut him the wild  dog and the cat and  said the wild boar ,  replied the boar , and  it is dead for my  master . you are not  too long as that that  the wild boar , and  it are dead . the  man ran out and cut  it and the wild boar  which were all standing and  that it was long as  a little dog and the  cat , and that was  not so long as the  cat , the man was  so long that it was  so long that it was  so long that the wild  cat , and the man  came to the bed and  began to cry and the  little tailor , went out  and cut a bit which  them that the dog ,  however , had not no  little rest for them and  said the wild boar ,  said the boar , and  have no one to be  my wife . the man  , however , had not  long long for the wild   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 136)\n",
      "(78).(30).(69).(92).(39).(51).(93).(10).(80).(22).\n",
      "Average loss at step 137: 2.802380\n",
      "\tPerplexity at step 137: 16.483824\n",
      "\n",
      "Valid Perplexity: 79.29\n",
      "\n",
      "Generated Text after epoch 136 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " hold , and the king ,  how not no one ever  be my wife and my  father , '' said the  little hare , `` that  is my bride , and  my child , '' and  his wife and him and  the king 's son went  home and said , ``  take my wife and god  that my child , ''  answered the king , ``  that do you come and  see that my father ,  god have no one ever  say that my wife ,  until that your wife come  with me or take your  wife and take my wife  the little wife and my  little wife and my wife  and and the little hare  says , how do you  not my wife and my  father , '' and the  angel which the king were  full of gold . and  his bride went home and  said , how do you  have my bride , and  my wife , and the  queen which my father are  great stone . and that  it is not my wife  and you have my bride  or wife . the girl  said that the king ,  said , `` that is  my bride , and you  will not be my bride  or not that you are  all my wife and you  take my wife and my  child , '' and the  king 's son and the  bride said the thread ,  and that the king had  him for this , the  little hare says that the  king , where the boy  says , how do you  take my wife and my  father and you do not  know that the little hare  is not a great deal  and rest which my wife  should not be a fine  deal of money . ''  it went home and went  to the little house and  cried , how do you  have my bride , and  your wife have a great  handsome one . '' the  king and this all the  son the queen had the  wife which the king ,  how not no one ever  be my wife and my  father . '' the girl  asked him that the little  woman had him . and  he went to the king  's son . the girl  rejoiced the queen , and  at last the child had  a rest which the queen  came home and said ,  how do you come and  do my bride and the  king and not a great  wild deal which it .  how do you come and  you do my wife and  you do that my wife  , and that my wife  are to be my bride  once so than if my  child , and not not  no one . the girl  did not know that the  little hare says , you  have my wife , and  you have my bride and  my bride , and the  little hare says the queen  that it is not a  little deal of money .  '' the girl went home   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 137)\n",
      "(2).(83).(53).(19).(58).(78).(5).(67).(79).(13).\n",
      "Average loss at step 138: 2.890328\n",
      "\tPerplexity at step 138: 17.999221\n",
      "\n",
      "Valid Perplexity: 76.49\n",
      "\n",
      "Generated Text after epoch 137 ... \n",
      "======================== New text Segment ==========================\n",
      " . then the king said ,  i have not for money  . the king , however  that should not be so  long as that the man  would not have an end  . it is not a  duck . the man would  have an king , and  which are the bride .  and the youth says the  child , and not a  long money for him ,  they will not be hold  of a great stone and  he must not have not  an money , but the  king says , what my  father , are you for  a little deal which have  come home and be my  father , you have no  rest and it that you  shall be my wife and  which are you . and  that as the man says  of him , i have  the bride be my wife  for a little stone which  had not fall on it  , and that the child  says , i have no  little work in its treasures  which your wife be a  little wife , but you  do not know what it  have no wife , and  do you know that you  are so long as you  come . and the youth  which the son and the  king which were a great  stone and the young king  would not be set out  dead , and went home  to the king , and  tell him that it had  the king home in his  mouth the thread and which  no little daughter , who  are no wife to be  hold that it had no  wife , but the youth  said the child , and  that the child had not  not an end , and  tell him that the king  , who were no one  them so long that it  was not long as it  that the king , who  came the king and have  the money which it .  it was no one any  one for her little life  . the king said that  they had not an end  in the world , and  that it had not no  little coat for , and  that the child had not  fall in the world and  see that it had not  the king , who had  not a great deal which  which should be be so  small until it that the  child , who had no  rest and it it so  long as it should not  have no one for it  until the king 's daughter  , and it had not  an end of the wood  . the youth said ,  no , you are the  king , and not no  wife , and not a  little deal of anything .  but the child says ,  how do you come and  you do no one ,  until that you are the  king to be a stone  . and that is the  man , will you be  so fine than you have  it still my wife have  my rest , and the  man says , you are   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 138)\n",
      "(21).(10).(52).(25).(19).(2).(20).(58).(61).(55).\n",
      "Average loss at step 139: 2.907149\n",
      "\tPerplexity at step 139: 18.304543\n",
      "\n",
      "Valid Perplexity: 78.68\n",
      "\n",
      "Generated Text after epoch 138 ... \n",
      "======================== New text Segment ==========================\n",
      " . but the king , who  did not know what he  should not hold that the  thread , the king and  daughter which he had not  such a little draught and  put the little home in  an ground . the man  's daughter , and he  thought that the old man  had not no one any  for them . the king  began to cry and the  queen of the king began  his cry . the king  , however , did not  find it . the king  asked what the king ,  but he did not know  how to eat it ,  or i will not be  a cause of the water  , and you will be  your wife , and will  take it to my father  until that the king ,  answered the king , and  i will not hold my  money and my wife ,  or you will not be  so so that the little  man is not a little  fellow , and the king  , who had no one  anything for the king ,  which are the king 's  young daughter . what a  little fellow of my wife  which are the king and  carry the boar home and  be my little wife ,  but you will not be  so so that the little  table is not for a  sack . the king ,  and the king , and  it must not hold it  in his bed and cried  to him , i will  no one ever shall be  my wife to be a  great deal one for the  water and see what it  should be as long as  that the king , who  must not hold that ,  said the king , and  did not know what the  king which had not no  one any one to be  it . and the king  of this the young daughter  said the thread . the  old man went home and  told him that the king  , who had no one  anything to eat and drink  . the man , and  that it had not no  money for his wife ,  who had no one any  one for him , and  delivered them in a little  which the young bride said  that it is not as  long as it shall be  an bride , and the  youth which he had to  eat . the man said  the cry . the king  , however , had no  wife the thread , the  wolf said the thread .  he went home and cut  it and the king ,  for that the king also  which had no one ever  live for them . and  the king said , how  do you come and your  beautiful man which is a  wife , and will not  go home . the man  said that the king is  full of gold . and  that the king also also  the son , and the  king also and no one  for them and be a  little one that it should   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 139)\n",
      "(33).(8).(39).(18).(48).(85).(38).(58).(4).(91).\n",
      "Average loss at step 140: 2.944247\n",
      "\tPerplexity at step 140: 18.996350\n",
      "\n",
      "Valid Perplexity: 75.59\n",
      "\n",
      "Generated Text after epoch 139 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " near him , and the king  said , `` that i  will take it home and  i have my bride ,  and i will take you  to my human shape .  '' the miller said ,  '' yes , and the  king and the king had  nothing but the old man  . the man , and  he at last the old  old women were full of  wood , and when the  girl saw the old man  with it to the king  . the king 's son  and the old woman had  to eat , the king  asked what the king was  the king , and the  king , and the old  maiden , and the king  must not hold it ,  and told him the old  man , the old witch  were full , ate the  old woman , and the  maiden , and the king  went home and went and  and the king 's son  and the old woman was  all a little of the  water , and the king  must not hold the little  of anything and said ,  take it home , but  i will not take it  home . '' the girl  with the old woman ,  which he had to eat  to the old man ,  the king , however ,  should not hold it ,  the king said , ``  my child , what do  i do that that i  will take it home and  i have my bride ,  but i will take you  to my human form .  '' the man said ,  neither now and if my  own wife live home and  do it and do you  that . '' the girl  gave him the old woman  with the old man ,  but the king said the  child , and that the  king said , `` that  i do not know what  you have . '' the  king said , `` what  are you here , but  i will take you home  or not my wife .  it are you for the  fine time , but i  must take it home and  be my wife and my  wife , and will not  take it home and you  will take you home and  bring my wife to eat  . '' the girl said  that the old witch had  the beautiful woman , and  one of them began to  cry and the king 's  young son said , ``  do you do that my  daughter , '' said the  girl , and that the  old woman had to be  an old woman , and  his wife looked at each  dress and that the king  and the old woman had  no one to eat ,  the king said nothing if  that are my bride ,  but i will take you  home and take you and  keep my wife and take  me home , '' and  that the king had to  eat , the young king  had him a ring which  was so beautiful that that   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 140)\n",
      "(71).(99).(98).(65).(26).(35).(7).(4).(33).(43).\n",
      "Average loss at step 141: 3.232873\n",
      "\tPerplexity at step 141: 25.352400\n",
      "\n",
      "Valid Perplexity: 77.87\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 140 ... \n",
      "======================== New text Segment ==========================\n",
      " who is not my bride to  be my wife . the  little tailor and the queen  . the queen , however  that should be set out  on the bed and said  nothing , and the little  man said , that is  not you and that my  master , i have not  a handsome man , and  the queen , and the  fox and the queen ,  the queen , however ,  did not find it and  the queen , and the  young queen was not so  long for this that the  little man had to be  hold that , and the  young king was so handsome  the queen , and the  queen had not not hold  her a little stone and  that the little tailor was  full . the little man  rejoiced and went home and  went home and told the  fox that the little tailor  and the queen , and  that he had not a  long time in his life  , but the little man  took the fox and the  queen and the queen ,  however , did not find  that , the queen ,  cried the little man ,  until the queen was not  a handsome man , and  in the country , and  that as a little tailor  and the little man was  a handsome man , and  when the little man was  full of wood , and  when the little tailor began  his cry , and in  her anger , and the  man began to cry ,  until the little man had  just not not a handsome  handsome one and the queen  and the queen , and  that the queen had not  rest in the world and  said , i am my  bride and not not that  she is not my bride  again . the man went  to the table , and  carried home and the king  which he did not know  for his wife , and  how the little man was  sitting in a little glass  which she did not know  that , said the man  . the little tailor and  that the queen had not  the queen , and the  old man 's son .  the man and the queen  began to cry . the  queen , however , had  not a handsome man ,  and the queen began to  eat and drink , and  he went home and told  her the little man and  he went home and said  the child , that the  young queen did not know  what the child , and  when the little man had  the queen heard that ,  cried the little man ,  the man and the queen  of the queen 's son  when the little tailor began  to cry , and the  young queen and the queen  again and the queen ,  too , said the little  man , and said ,  that do n't be my  daughter , but that is  no one , and not  the queen , and the  fox said , the little   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 141)\n",
      "(55).(93).(48).(40).(95).(39).(24).(45).(6).(21).\n",
      "Average loss at step 142: 3.022491\n",
      "\tPerplexity at step 142: 20.542390\n",
      "\n",
      "Valid Perplexity: 78.09\n",
      "\n",
      "Generated Text after epoch 141 ... \n",
      "======================== New text Segment ==========================\n",
      " , said the king , and  the king said , what  have you not no one  for him , and not  that you have no one  any one for him and  said the king , and  that the king is so  long that it is not  no one it . the  man said , what do  you be my wife and  cry and you can not  know that , answered the  maiden , you are a  great deal of money for  the two wife . the  old woman did not know  what that would not be  still so beautiful and this  the king did not know  for him and said ,  now that is not a  beautiful horse , and the  fox and the king and  the king 's daughter ,  which he was so well  . the king , however  . the maiden , and  that the king is so  beautiful that you do not  know what you can not  believe , i are so  so long that it is  still so beautiful that you  can not know that ,  replied the king , this  that a great deal which  it who should be set  home , said the king  . what do you come  for me , and the  king 's daughter , and  said , i will not  know that , answered the  maiden , you are the  beautiful maiden , and the  fox which the fox let  the fox get her and  put him home and put  it so a little little  man and the king had  to sing it , and  his king and what the  king had to eat ,  and the king was so  so that the king had  to sing that , said  that the king , who  is a great deal which  it of them and the  maiden and the king 's  daughter , the king ,  however , should not be  set out in the wood  , and the king ,  and the king 's son  went home and said the  king , this , answered  the king , and the  old woman did not know  one word which the king  was so beautiful that the  old woman did not know  one word which the king  would not know that the  old woman did not know  one word of the king  and the king 's son  was so beautiful and what  it would not know that  and said the king ,  i have no one for  me . '' the girl  did not know that the  old woman did not know  one word which the king  were so beautiful that the  devil had not an end  as that the king was  sitting in the forest ,  until the king had him  the great deal one who  came home and said ,  i will not be my  son . what do you  not know for it and  bring it home and do  not know that it is  no one , and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 142)\n",
      "(86).(13).(73).(33).(80).(5).(77).(63).(12).(53).\n",
      "Average loss at step 143: 2.895053\n",
      "\tPerplexity at step 143: 18.084467\n",
      "\n",
      "Valid Perplexity: 78.97\n",
      "\n",
      "Generated Text after epoch 142 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " in her way , and the  whole house which had no  rest and a great deal  one them so a little  horse and the horse said  that no one are my  son . the man says  to the father , you  will take your horse home  or you , and the  little tailor which is not  a great deal which is  not my wife , and  that is a little dog  the little house which is  not my wife , and  i must not know that  , answered the father ,  i must not have it  and do you come and  keep my horse and and  you can not know it  and do not know that  it is my wife ,  until it are all a  dog which is a little  horse . the little tailor  says , i will not  know that , answered the  king , this is a  great deal , but that  my father is a horse  . but the father says  that the father is a  stone one , and the  little tailor which is not  a little deal which is  wife and my wife ,  and the horse deal which  you can have for a  dog , i will not  be my wife , and  that you can not know  it , answered the father  . the man says ,  i must not have it  . the little dog which  says to him , i  can not know that ,  cried the father , and  is a great deal which  can be my wife and  take my life . the  father says , i will  give you a little deal  money and do you know  as long as you come  for a great deal one  in a house and which  is my wife , and  it shall not be my  husband . the man says  to him , i have  my bride , and the  father which is a horse  . and the master says  that a little man is  for the little dog ,  or you will not be  my wife , and you  must be my wife ,  which you can not know  the little dog and the  whole forest , and the  man says , i will  take you my wife and  see what you can have  that , i will not  be my wife , and  i must not have my  human shape . and the  father horse it , and  gave him a great deal  one in the forest .  it was not no one  to be a horse and  said , i must not  have it . the little  house says , i will  so that you can not  pay it so that it  can be my wife .  but the master says ,  dear father , you are  my husband , and you  are my husband , and  i must not have my  wife , and you can  be my wife , and  goes home and the little   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 143)\n",
      "(89).(11).(59).(36).(3).(88).(8).(92).(32).(23).\n",
      "Average loss at step 144: 2.816181\n",
      "\tPerplexity at step 144: 16.712897\n",
      "\n",
      "Valid Perplexity: 83.46\n",
      "\n",
      "Generated Text after epoch 143 ... \n",
      "======================== New text Segment ==========================\n",
      " in her life . hans said  that no one would not  die . what do you  live . what do you  give you , i will  be my bride . what  do you have for the  wedding . what do you  give you . what do  my child , hans .  what do you do .  what do you do .  what do you do .  '' the girl said ,  i will take my life  . what do you do  that you do not take  her . what do you  take for your life .  you do not want it  so that you do anything  until me your wife ?  what do you do .  that do you do my  life , but that you  must take it home .  '' the girl , however  the daughter , and the  king and death , and  bring the little dog .  you will take me home  and take my life .  you do not know that  you should have my wife  that you do . what  take you , and bring  my life . what do  what do you do .  or you are you for  this time . what do  you do . what do  what do you do .  that do you do it  for you . you do  no one that you should  have it . you will  take me home . ''  the girl and the king  's daughter home . the  little tailor , hans .  how do you do that  you do you that that  my father , mother ,  i have no one ,  or that that i will  do it . what do  you do . what do  what do you do .  or that your wife are  my wife , and you  must take it home .  what do you do .  hans , '' said the  man . what do you  do . what do you  give me . what do  what do you will ,  which do you will take  me . what do you  give it . what do  what do you do .  what do you do .  i bring that you should  have my wife , and  that i will take it  into my bed and do  die . do you .  the little dog which had  rest which it should be  all . she did not  have no one , and  bring the little dog and  go home and fetch you  . what do you do  for your life , but  you will take my life  to take my life .  what do you do .  '' the girl said ,  `` do you do that  you have no one anything  to take my life .  you do what that do  that , the child is  still . what do you  bring that it are my  rest , and that it  is no one , i  must take it home .  what do you do .   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 144)\n",
      "(81).(18).(80).(85).(74).(44).(87).(0).(65).(64).\n",
      "Average loss at step 145: 3.140900\n",
      "\tPerplexity at step 145: 23.124672\n",
      "\n",
      "Valid Perplexity: 76.85\n",
      "\n",
      "Generated Text after epoch 144 ... \n",
      "======================== New text Segment ==========================\n",
      " daughter to her wife , and  it was not so long  to the bride , who  was so so as it  should be an bride and  his wife , and the  bride said , i have  not for my human form  until the sun , and  you have no one in  the world and bring it  so that you do not  do you that you have  my bride , and you  will not be my bride  , and i will not  be my bride , and  you will not be my  human being . and that  a great feast which had  been so beautiful as the  old king , and the  sun had not no one  rest a glass , and  this morning the king had  no rest , and the  two horses , who was  for an king 's daughter  with their human form and  bring it home and bring  you home and bring it  to light . and that  the king had to be  once like the bride ,  she was so so that  that the king had not  an bride who was to  meet her , and the  bride said , i have  no human being that ,  i have no human being  that , '' and the  tailor said , i will  not be my bride ,  and i will not not  my human form and bring  my life and bring the  glass home and bring it  home and do it for  my human form . the  devil 's daughter and a  long time , and the  king's daughter said , i  must be my wife and  the king , and the  whole sea , and that  you will not be my  son , i have no  rest which that you that  . '' the aged witch  were so beautiful that the  old king had not an  bride who was for the  whole time , and the  king 's son had not  come home and see that  it should not have an  king 's daughter , and  that that the king was  a white dove , and  that the king had no  human form . the king  again and the bride went  to her bed , and  her white white horses which  had rest which the king  did not know that the  young king was a bride  with her heart , and  at last the king had  the bride let him eat  . the tailor said ,  what have you be my  son , or that i  will not be my bride  which i will be my  wife , and you have  no one , or you  must be my wife and  work . the bride asked  him that the king 's  young son had to break  into a glass and the  young king said the king  . what have you not  be my bride , and  it is there . ''  that now that is a  wicked dove , i will  go home and fetch you   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 145)\n",
      "(13).(77).(20).(34).(69).(91).(1).(38).(17).(82).\n",
      "Average loss at step 146: 2.977779\n",
      "\tPerplexity at step 146: 19.644145\n",
      "\n",
      "Valid Perplexity: 73.07\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 145 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " away with the gold which the  sun had not no one  any one for her and  see that it should not  get out , and had  not an end of a  gold and which he was  in great sleep . the  little tailor was just so  a long long one could  that it should be an  old woman 's son ,  she was not so as  he had no human form  for the king , and  give her to return to  the door which had no  wife any one and be  all . and that the  old woman had to be  laid on the cock ,  and went to the door  and the daughter and the  door , the old woman  had her son , and  at last the king rejoiced  the king , and the  whole castle , and had  her daughter to her .  and as the king had  her son and the son  which had passed and the  king began to stop the  bed , and himself himself  to an end , and  her father gave her all  to the castle which the  door was the bride and  but a handsome man ,  the king , however ,  did not know how to  go and cried , i  will not go home ,  or not that you are  so long that that the  little girl would not be  love , and as the  whole house that , she  must not get out of  the door and he had  not come to light .  the man said that the  girl had not come and  get her so long and  said , i will not  be my bride , and  have no one any one  or my wife the bride  which i have the bride  your wife , and that  my father would not be  no one for her wife  . the king went home  and just as it had  an bride come and see  that the king was so  white that the king was  a handsome man , and  when the king had the  castle that the old king  's daughter herself and told  the old king , and  just as it was all  full of gold that the  old woman had her daughter  . the king 's son  did not know how to  go home , and the  whole house which she had  an bride daughter with her  one and the king 's  son rejoiced and went and  cut her life and the  cock of her own bed  that he was to be  an old woman 's son  . the king , however  that had no one any  one for him , and  that the king was so  white as the whole castle  and cried , i will  take you home and take  my life home and take  me home and take it  home . the little girl  had not come to light  . the king went home  until the cock on the  door , and went home   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 146)\n",
      "(28).(79).(49).(78).(42).(25).(57).(93).(61).(35).\n",
      "Average loss at step 147: 3.156531\n",
      "\tPerplexity at step 147: 23.488974\n",
      "\n",
      "Valid Perplexity: 75.98\n",
      "\n",
      "Generated Text after epoch 146 ... \n",
      "======================== New text Segment ==========================\n",
      " to be a great deal which  things , or not my  son , '' and the  cook , said the woman  . `` that is a  fine bird , i will  not see my father and  you have my wife .  i am so clever that  i will not be so  man as that as my  mother is so beautiful that  i will not see my  mother , '' and the  maiden said , `` what  your little sister , i  am my son , ''  said the woman , ``  i will not see my  wife and my father ,  i am so beautiful that  i must not be so  so so that , you  i will not see it  . '' the man went  into the door and lighted  until the sun was so  red that the man had  certainly come home and see  and the house were so  beautiful that if it must  get a little deal ,  i will not be my  son , '' '' ``  yes , '' '' said  the man , `` if  your father have not for  the bird , '' and  a little bird which is  certainly my wife , and  it must not be my  father , '' and he  did not know what a  golden bird and the bird  which is certainly a little  bird , '' and the  bird , said the woman  , the house is a  beautiful bird , and the  whole stones which she must  come home , and the  father said , `` my  little sister , will you  be my bride and the  bird , '' '' said  this , `` that is  an man who is not  my wife , '' and  that he had not him  it for a great stone  that he ought to see  how the house were all  so beautiful that it was  not a man and the  little bird which is certainly  my wife , and the  man said , `` what  a little fellow and my  child , '' and the  peasant said , `` what  do you have for the  sun , '' '' and  he went and put him  in the house and went  so so a little man  went home and went home  again , the man said  , `` that is a  fine bird , i am  my son , '' and  his father said , ``  and that my child ,  i am so beautiful that  i will not see it  . '' the man said  , `` that is the  old man , '' and  went and went and the  top of his heart ,  it had no one any  need to return to the  table , and went and  until the king , however  , went and the door  on the door which were  a house . the girl  went home and went and  at the same time the  son said , `` what  a little fellow , ''   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 147)\n",
      "(96).(15).(84).(87).(5).(71).(43).(88).(92).(12).\n",
      "Average loss at step 148: 2.980942\n",
      "\tPerplexity at step 148: 19.706377\n",
      "\n",
      "Valid Perplexity: 76.53\n",
      "\n",
      "Generated Text after epoch 147 ... \n",
      "======================== New text Segment ==========================\n",
      " been quite a handsome man and  so long as the little  bird which had no one  life , and the king  's son , and the  girl had the daughter to  be so long as the  old woman was so handsome  a little and the king  , who had no one  for her life . the  man said , no one  will not be my wife  for me , you go  home and die , or  will not go home and  fetch you a little little  deal and which have no  fear , what not my  human be so long as  that a little man will  be so handsome that that  it is no one for  it . the girl ,  said the king , that  do not be my wife  for me . you will  go home home home and  bring it home and you  do my wife wife ,  or not that you have  my wife , and that  that the little woman is  so handsome that the little  man said , no one  will not be my wife  home and you will not  be my bride , and  shall not be my wife  for me , you have  it so that you do  my human form , and  have no one for them  my wife , and that  you do not know that  my wife are no one  for her father . the  soldier was so so that  the little man had no  human form and bring the  three ring and the little  little dog which he come  into her sleep and the  queen with her wife and  the king and the king  , and the king 's  husband , and the king  said , i will not  be my bride , and  shall not be my wife  . and the king was  a handsome woman , who  knew that a little man  could not come home and  be my wife and the  king , and the king  's daughter , and went  into the chamber and the  daughter and the king ,  and the king , who  did not come home and  be so so that the  little woman had the wife  whom the little woman was  full of four pieces which  which had the wife be  an king , and the  king , who had no  human form . the king  the son went home and  found it to her wife  which the king was so  so that the little man  were laid in his work  or not that no one  would not come home and  bring the little wife and  her life , and the  two others which the king  's son . the king  also and the old woman  that the king , how  not no one for them  for the old man and  bring the ring home home  or you . the girl  and the king which was  all in the chamber ,  until the king was a  handsome woman , who were   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 148)\n",
      "(6).(43).(53).(90).(24).(19).(68).(67).(10).(93).\n",
      "Average loss at step 149: 2.908419\n",
      "\tPerplexity at step 149: 18.327802\n",
      "\n",
      "Valid Perplexity: 82.75\n",
      "\n",
      "Generated Text after epoch 148 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " before her great stone which had  laid it so a great  glass which that the three  man were laid to be  like so so that they  were not laid in his  pieces and kissed them and  cut his head them and  a great deal which were  and straw , and were  very clever , and it  so so one as it  as the old woman was  so very so that the  old woman were so clever  because the king , son  were so clever , they  father all it fell so  great a great deal which  which which the queen had  such a great deal which  that the queen was so  so that the father said  , `` dear son ,  have not my bride ,  and do not know that  your father says , how  do you come so well  , and do not know  how to do , ''  said the father , how  do you come home and  do my father , ''  '' said the father ,  how do you do my  bride , and do not  come home . '' the  next day and this as  soon as the last man  and the son and the  queen also and the one  fell home and cut a  glass so glass and a  glass that it that the  three man were full of  house . the last one  as that as it was  all so so that the  old man had such a  great deal one for it  until the king , how  only the king , how  have you my wife ,  i have my bride ,  how do you come or  live , or not that  your wife have such a  fine deal which money .  '' the girl went home  again , and as they  were so very so that  they should come home home  until the king 's son  and the son and the  queen also and the one  was so so that the  old woman was great feast  until the king , son  began to cry , and  all one one and the  queen which the old woman  were so so so as  as that he should not  come home and said the  old woman , `` and  this is my bride ,  or do , '' and  not one one and the  king 's son , and  that they had no one  in his life and let  them do how no one  ever have such a glass  one one and my wife  that you have no one  for them , and that  is a great deal which  things which i come home  until my wife . ''  this the third , the  second came and the last  day that they were so  beautiful until the old woman  had not one one and  the king 's son ,  until the third said the  queen , how do you  not my bride , and  are my bride , and  shall not be my wife   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 149)\n",
      "(82).(73).(37).(81).(72).(57).(52).(30).(10).(89).\n",
      "Average loss at step 150: 2.828218\n",
      "\tPerplexity at step 150: 16.915295\n",
      "\n",
      "Valid Perplexity: 84.74\n",
      "\n",
      "Generated Text after epoch 149 ... \n",
      "======================== New text Segment ==========================\n",
      " to pass that this the mother  said , how do you  go or your wife that  this do you come home  , or not my wife  should be touch my wife  for your life , and  how do you not my  bride , and do not  come home . that the  child had rest beneath the  window . the girl ,  how no that that do  it like my wife .  it is no one that  it was no one in  his bed and and the  king 's son , and  how no one shall do  it that , how the  wife says , how the  father says , how no  father do not touch it  , and at last the  child had rest and a  and bed , and the  king , how the child  would not come home .  it at last the child  had rest which it the  queen and rest and the  queen , and the child  had rest and all the  heaven which it in a  glass which that it was  not all to do the  father which the bride had  the wife , and the  mother said , how do  what no one touch your  money . that no one  shall be my wife .  and the son herself at  this the child had rest  beneath the ground and said  , how do you come  so that it shall not  be my wife for me  . how do you do  for your fortune , how  not my wife ever be  your wife . and that  this this do not come  home . that that no  man , how do you  live like my wife ,  or how do you come  my wife , and let  her be my bride .  how do you do for  my life , and that  do you come home and  die . it that do  it for us or do  . how it do not  that that i have no  one for them , and  it at last the child  had rest which it the  daughter , and the child  had rest and pieces and  if the child had rest  speed and that the child  had rest and a great  straw , and how no  the wife had not rest  for them , and it  at last the child had  rest and a bed and  it at last the child  got the child and this  that the queen had rest  beneath the ground . the  man went and until it  as the bride the daughter  which the child had rest  beneath the ground . the  mother said , how do  what do my bride have  her wife , and that  you do not forget it  like my wife , and  how no that shall you  go home . the girl  was so beautiful that it  came all her until the  bride , and the child  came home again and the  bride went to the ground  . and that is no   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 150)\n",
      "(62).(33).(15).(83).(50).(73).(1).(94).(60).(9).\n",
      "Average loss at step 151: 3.111144\n",
      "\tPerplexity at step 151: 22.446708\n",
      "\n",
      "Valid Perplexity: 75.15\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 150 ... \n",
      "======================== New text Segment ==========================\n",
      " who would not believe that the  queen had no one ,  the king 's daughter went  away and said , what  be my bride , and  will not be my wife  for me , you may  take her little wife and  take the little bed and  my bed , and the  king 's son went home  with the king , and  he went home and told  the king , and the  queen said , what do  n't do that that i  know that that is no  end of them . the  girl was so beautiful for  a great stone which the  old woman had no one  for her wife . and  the king said , what  have you come and your  wife be my bride ,  or you are no one  , and that you are  no one for me .  it is no one for  her wife . and the  third said the queen ,  said the king , and  how no one shall be  your wife . and that  as the king had no  other dress , she had  an old woman 's son  came home and went home  until the king came home  until the king 's daughter  in her bed and said  the child , and that  that the king was so  handsome that the king was  full of joy , and  the king 's daughter went  and the queen went home  and went home and told  the old woman , the  king , however , should  rest in a bed and  bed them a little dress  . the king said ,  how do you come and  my be my wife and  that my child , what  i do , and that  is , you may be  my son and you shall  give it a great stone  , the king answered ,  that is no one ,  but that is no one  that it shall be my  wife . the little tailor  and the son , and  that no one had a  beautiful bride , and the  two sisters which she had  such a handsome one and  dress , and the king  said , what do you  do , and not no  wife ? the girl had  the son , and the  old woman went home and  set her a great stone  and which had been so  rest for her heart and  said that the king 's  daughter was a queen ,  the king went home and  told her how the king  's son had been so  rest for her heart and  take a little bed and  rest which it such a  long time , the king  said , what do you  have , my child is  , that do not be  my wife , and that  that no one shall be  my wife . the little  hare began to cry ,  and as no one could  go home and said the  child , and that the  queen had not be a  old man , and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 151)\n",
      "(22).(47).(7).(88).(61).(60).(37).(31).(23).(86).\n",
      "Average loss at step 152: 3.015736\n",
      "\tPerplexity at step 152: 20.404093\n",
      "\n",
      "Valid Perplexity: 78.82\n",
      "\n",
      "Generated Text after epoch 151 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " UNK . `` that is no  one be my wife .  '' and when the tailor  heard that , the tailor  ran down and the tailor  again , and the princess  came home and cut it  in a bed and the  king , who was so  as that as a little  woman and the tailor ,  and the tailor went into  the stable the little tailor  fell on the fire ,  the window and laid it  for her wife for the  table , and the tailor  fell in a bed and  bed for her . the  second was for a little  man which she was to  drive home home and the  girl was full , and  as the tailor heard that  for the tailor , and  he went and put her  little and the little light  once in the bed and  heard that and that the  girl , who went and  cut the light and the  tailor , and she went  home and threw it into  her and the tailor ,  when the tailor saw the  man and the king ,  however , should not be  too so that it was  not so long as it  fell in his bed and  she put the little light  for her little having life  it for her and the  queen , and the tailor  heard that . the tailor  ran down and went to  it , and the tailor  began to speak and went  out and cut it a  bit which it was so  handsome , the tailor and  when the princess heard that  , the princess heard that  , and ran after her  and threw herself on his  head and joy and the  believe , the tailor ,  he went home and cut  the fire which it had  the queen in his bed  . the tailor went home  with her , the tailor  began to speak and went  and at each other and  put the little light and  its bed and the tailor  fell in a bed and  bed for her . the  soldier went and cut her  great stone and which was  all to pass that the  king , and it was  all in a great wood  that the tailor had the  king , and it had  nothing for her . the  second was the queen ,  and the tailor was to  drive home and see that  that was a man for  a great stone which had  fall in a bed and  it had a little man  in a great joy and  all a little man which  was a man . the  girl 's all as it  did not believe that she  would not believe that ,  said the tailor , ``  do n't do you that  . '' the tailor saw  this , the tailor ran  away and threw her a  fire and which it in  great joy , and the  tailor went home and threw  the little light and a  great stone which was in  heaven . the man was   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 152)\n",
      "(36).(95).(50).(67).(60).(4).(94).(24).(76).(16).\n",
      "Average loss at step 153: 3.075897\n",
      "\tPerplexity at step 153: 21.669305\n",
      "\n",
      "Valid Perplexity: 77.55\n",
      "\n",
      "Generated Text after epoch 152 ... \n",
      "======================== New text Segment ==========================\n",
      " and said the young young young  woman were the queen ,  but the young king said  that the king 's son  did not know how to  come home , and the  king's daughter said the young  queen , that that the  old woman said the young  horse , which is no  one for it . the  king 's daughter said ,  no , we are the  bride '' and the third  the young young thread .  one of them came and  an king 's daughter and  how it should not know  one word of it ,  she said , `` what  that that is no one  , but that is no  fear , that will not  pass that , but that  that the devil said the  king , and that that  she would not be no  other than it for her  two children and which had  no one sleep for them  , the young man would  come home and be my  bride . and the young  witch did not know what  the young bride were not  like no one , but  no one who will be  my bride , and that  will be no one to  be my wife . the  king 's daughter said the  fox , and that the  queen had not not be  no one in his life  , and the young king  went home and begged for  his wife . and the  devil said that it should  do that it should be  an bride and the king  and the end of the  king , who had no  one them in his bed  and said the young young  grandmother which had come home  again , but the young  king bore it , the  two children flew home and  laid it on his bed  or the two children which  he was in his bed  . the king 's daughter  went home and begged the  mantle . the king 's  daughters were for a long  man , and the young  king said , what that  do that that you are  my bride to be .  that will not be my  dear bride , and that  no one will be my  children . the young young  devil is to be my  silver wife . the devil  cried , `` you are  it dead , and that  you will be my bride  . and that is no  than death . the young  witch bore his mantle which  the daughter said the young  woman the king 's daughter  flew home and said the  king , how that that  no one will be my  bride , and that that  he should not be no  rest until it shall be  your wife . the king  had not hold him up  for a great stone which  he knew that it was  all in his sleep and  come home , and the  cook , and the young  queen flew home , the  king 's daughter seized it  for a great stone which  sat down in his bed   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 153)\n",
      "(33).(51).(7).(18).(4).(79).(26).(59).(28).(42).\n",
      "Average loss at step 154: 2.867226\n",
      "\tPerplexity at step 154: 17.588160\n",
      "\n",
      "Valid Perplexity: 78.92\n",
      "\n",
      "Generated Text after epoch 153 ... \n",
      "======================== New text Segment ==========================\n",
      " , that no one would not  have an old man and  it for you . i  am my bride , and  we will not be still  as that that i will  give it home and you  be my bride , and  that no one are my  dear wife , and the  devil said , i have  nothing for him , and  it will not be my  wife . the girl would  nothing for him . and  he had nothing to be  so long , and was  no one any one for  the two life and the  king 's daughter , and  that the old woman did  this that the girl would  go home home , but  a great glass things who  had the bride be a  king , and no one  will be my bride ,  the children would not be  an old woman and the  king 's daughter to life  with him , and the  old woman took her life  home . the girl took  it down to her and  threw a great glass which  it in him a great  deal which it was a  great glass , and the  young girl had and no  father in his bed and  said , i will not  be my bride , and  we will not be still  as that that i will  not do it . i  am my bride , and  you will not be my  wife , and i am  my bride to be my  bride , and i will  give it home and do  my human form . what  shall you be my bride  until my father , and  that you will not be  no one , and i  will not have you for  it , but you will  go home home , and  not my bride , and  i know that you shall  carry him to my little  palace . what do you  be my bride , and  how the father would not  come home . what my  bride have not , but  my child , i am  a little creature , and  gold be my wife and  that my father would not  do you , and the  girl said , i will  give it home and do  n't be my wife ,  which i will not not  do it , and the  two sons , and she  had nothing for him ,  which would not be still  too that it should be  an bride , who had  the king home home and  do and i will not  have my wife , and  it will not be my  little bride . that that  i will not be my  dear wife , and i  am my bride to be  my wife , and i  know that my child ,  that do n't be my  dear wife . and the  old john did not know  for him for her little  sister , will you be  no little one that it  shall you be my wife  until you will not be   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 154)\n",
      "(50).(53).(12).(22).(31).(38).(11).(14).(55).(73).\n",
      "Average loss at step 155: 2.981891\n",
      "\tPerplexity at step 155: 19.725074\n",
      "\n",
      "Valid Perplexity: 78.79\n",
      "\n",
      "Generated Text after epoch 154 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " , `` you shall not know  that that you have no  fear , you have no  so long as you have  so long that you have  my wife , and you  shall be my wife and  take my life and you  know that your father ,  '' and went home and  went home , and the  old man , who had  the son come home and  go home . the little  boar says , `` that  are my bride for you  or take my life and  '' the son said ,  i know that that that  no one shall n't be  your wife , and that  it is not my wife  , and you have no  fear , you have no  son , '' `` that  are you for my axe  and do you that you  , you shall give me  your strength , take the  bread my wife . ''  and the father said ,  how do you get so  beautiful that you have no  end of my life .  '' the father went home  for the father and the  king , however , went  once more , and that  it was all in his  bed and said , ``  how do you get as  well . '' the father  went home and went home  so a little deal that  you do not know that  your axe have for the  father . '' the father  did not know what the  old woman had the daughter  with him , the father  went home and went home  so a little deal that  that he could not be  so end that it had  such an end which the  wild meat , said the  woman , `` that do  not know that you shall  have it . '' the  son went home and went  back and said that the  king was a witch 's  wife , and that the  old woman had the wife  who who did not know  it , and the king  in the country , the  king , however , will  not know that , the  old man did not know  what it is for the  old man . '' the  boy went home and went  home and went home and  cut it home and the  cook himself and the daughter  , how that that is  an end , and you  shall be my wife and  my wife shall not be  your wife , and that  you do not know that  you shall be my wife  so long that you have  you for you , i  must take your life and  take your life . ''  `` that that that no  father is not my wife  or not that you shall  give me some light and  see that my father are  a wife , and you  will take the life and  take my life and be  no one for him .  the father , however ,  will not get home and  do n't be my wife  and my father , ''   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 155)\n",
      "(26).(7).(46).(1).(21).(11).(12).(55).(93).(71).\n",
      "Average loss at step 156: 3.029556\n",
      "\tPerplexity at step 156: 20.688045\n",
      "\n",
      "Valid Perplexity: 75.19\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 155 ... \n",
      "======================== New text Segment ==========================\n",
      " house that the king had to  come and see how the  queen did not know how  in the death , and  the king , and the  king , who were a  son , and as the  old woman had the daughter  and the king , who  went home to the king  . the king took her  all his wife , and  he went home and put  it so a great deal  or money . the king  and this the old woman  was all so that that  as the old woman were  full of water . the  king went home and went  away and put the two  other home home and put  it on the water ,  and the king , who  fell in his bed and  gave him the two children  in the water which had  the daughter to be so  long that they were all  merry and a little deal  of money and the king  with the death , and  now not not one one  , and the king 's  children , and went home  so a little man .  that the king had to  get home and the king  's daughter to be so  beautiful until the old woman  were still more until the  two others until the old  woman was full of glass  that the king had to  die , and thus the  king also appeared until the  woman was so beautiful and  until the old woman were  so clever , the old  woman were all in her  great pieces . the old  woman were quite still so  that the daughter was so  very long that the old  woman had no one to  eat and drink , and  the king , and the  others , and the king  of this the king was  a queen , and the  woman rejoiced the king ,  the others , who was  not so so that the  old woman had the daughter  to the death . the  hare said that the old  witch had not a great  fire , the two others  whom the old woman were  still content until their death  . the king rejoiced one  until the king , and  his daughter , however ,  should not believe that the  king had to be so  beautiful and all that the  queen had no one any  like the two one 's  gold and the king ,  and the king 's daughter  to be as a little  little deal , the king  told him that the king  also , and the king  's daughter , and the  king , and the king  who had not be so  most so so until the  king , and the king  's daughter to be as  it as no one ,  until they went home and  believe the old woman ,  that the king had no  son , the old woman  had the daughter , and  that the king had to  get home and the king  , and the king ,  which loved them the old   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 156)\n",
      "(10).(97).(65).(64).(32).(67).(43).(77).(85).(96).\n",
      "Average loss at step 157: 3.006548\n",
      "\tPerplexity at step 157: 20.217487\n",
      "\n",
      "Valid Perplexity: 80.12\n",
      "\n",
      "Generated Text after epoch 156 ... \n",
      "======================== New text Segment ==========================\n",
      " fisherman and the bride 's son  and the king 's son  . the girl 's son  when the king 's son  were full of glass that  , the bride was so  long that it was all  there . and the king  and the king 's son  , and she went home  and put it in and  put him in the glass  forest , and it was  the bride , and the  king 's son 's son  , and was a great  glass of bread and the  bride 's son and the  queen , and the king  , however , went home  again , and it was  a great glass of bread  until the king 's son  and the king 's son  went home and she was  so beautiful that the bride  said that the king 's  bride were home and was  there . the king 's  child , and went and  put it so a great  great glass which it was  so beautiful and this she  were so beautiful that it  should not know how to  eat it . the bride  was so beautiful and all  the whole night which the  queen had the bride home  . the next morning the  little man went home and  asked it , and the  bride said , `` you  are my bride , and  can not be my bride  , and you will not  have it , '' and  said the king and the  old woman , she was  very very beautiful , the  king 's son 's son  and the king 's son  was in the castle ,  but the bride said ,  she are a great deal  , and this morning the  girl went home and the  queen went home and just  opened the castle and the  queen 's son she went  in the whole forest ,  but she went home and  put it so a little  little house which had no  one sleep so it it  she was so beautiful and  how she should not hear  that , she went home  the whole night which had  the bride light . the  girl was so beautiful and  thread , the whole night  the little house and which  fell in the forest ,  it was a great deal  one which the king 's  child , and the bride  gave him a great deal  which which had the bride  them . the next morning  that the king 's son  rejoiced so a little man  , and had no one  in the glass which which  was a great deal which  should be glass so glass  until the whole night which  and the bride were so  great the queen , and  it was all so beautiful  the whole night , and  asked the girl that the  king 's son were so  little one the bride had  the bride and the king  's son . the girl  's son at last the  old woman came home and  it at last she went  home and put it so   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 157)\n",
      "(61).(5).(22).(49).(75).(11).(50).(72).(64).(60).\n",
      "Average loss at step 158: 3.228426\n",
      "\tPerplexity at step 158: 25.239895\n",
      "\n",
      "Valid Perplexity: 75.44\n",
      "\n",
      "Generated Text after epoch 157 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " she would not get on a  great glass which might be  an bride , who was  so beautiful that it ,  cried the angel , and  he went home and asked  that it should be as  as that as that as  no one knew him .  it must be as if  she would not be an  cause , and it must  not be too so as  they will not be as  that as that that you  can not find it ,  for you shall not be  no one for me ,  and the angel , i  will not have you for  you . '' and now  that came and said that  no one knew them ,  the prince said , i  must not be as no  long as it shall be  your wife . but the  girl said , i will  get for me . ''  and went and put the  whole sea , and went  home and put it so  until the door . the  bride went home and went  into the forest , and  at length , and went  in and said , i  must not be as no  dream , but that i  can not be as that  for me , but that  shall you come home and  eat , i will not  be my wife and the  water , i will not  be my wife and it  do not find it .  '' the duck had no  wife and the bride ,  where the king was still  tall until how he had  ever set out , and  told him that it should  not be an bride ,  where the king which was  all in the forest ,  which were dead . the  king , however , should  have no one for him  , but that is a  great deal of money or  hunger . the prince said  that it should not be  as so as it should  eat and drink that you  shall be my wife and  it , but that i  will not be my bride  for this that you shall  have it , but you  are my bride to be  too that you will not  be my bride , and  you will not be no  need in a place which  i will give you a  great stone which are a  great deal which which are  the bride , and the  moon , and the king  , who will not be  my wife and you have  my bride , and the  king said , i will  no one have my bride  that it , i have  my wife and you have  it . i will not  find it . '' the  girl said , i will  have it for you .  you shall not find my  money home and have you  . '' `` i will  get in a ship which  are an bride which are  it for you . the  girl said , i will  get in the whole world  , and if you have   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 158)\n",
      "(96).(92).(6).(27).(39).(68).(10).(2).(59).(36).\n",
      "Average loss at step 159: 2.909289\n",
      "\tPerplexity at step 159: 18.343756\n",
      "\n",
      "Valid Perplexity: 80.09\n",
      "\n",
      "Generated Text after epoch 158 ... \n",
      "======================== New text Segment ==========================\n",
      " two man 's daughter , who  will not learn what it  will be so glass .  this that the wild boar  and cut the glass water  a glass and half which  it was so long that  that the wild boar was  full and the fire ,  however , the wild boar  ran home and threw over  her full which it at  each other until the wild  boar was cut them a  great deal which it in  a hill which had so  half one for the glass  which the bride saw the  ring and the glass which  that the children had made  no one and the glass  until the wild boar was  over and said the wolf  that the king was a  wicked man , and the  king 's daughter was so  so so that the wolf  saw the wolf , who  fell in a glass which  which made the glass and  a glass and that that  he was not a glass  in half . this ,  answered the girl , the  dog and this i will  have the ring , and  now not go home and  fetch my axe , and  let it go and fetch  my axe , and cut  a glass glass which my  wife shall not learn my  end , i will not  find my wife wife .  it shall not learn that  that you will be my  beautiful wife . the girl  said , i will not  have it for my wife  that you do not know  that that no one shall  come home and fetch my  little wife and the glass  and my wife and the  ring which you should be  so glass , you have  my bride , and the  girl said , i will  go home home home and  fetch my axe , and  cat that it shall be  so glass , my wife  that i will be so  glass . it , you  and not that , the  dog and this , the  girl and one day that  they went home and kissed  them so one and made  all one one the thread  , the cat was so  so long that that the  dog , however , got  it but a little man  had to go home and  fetch the glass which the  wild ones was full and  thought the thread , the  wolf was so great a  handsome dove , and the  cook began to weep and  cut the hill which were  a beautiful wife and the  dog , however , the  wolf went home and cut  him a great noise and  until the cat , and  as as as as long  that the man was so  very long that the man  saw the ring which was  just as that that the  man had to find her  so long that the girl  could not cut her glass  and half which it was  so long that the children  saw it but the man  fell in a glass and  glass and the glass which   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 159)\n",
      "(79).(66).(96).(1).(56).(88).(32).(40).(49).(85).\n",
      "Average loss at step 160: 2.930279\n",
      "\tPerplexity at step 160: 18.732851\n",
      "\n",
      "Valid Perplexity: 77.69\n",
      "\n",
      "Generated Text after epoch 159 ... \n",
      "======================== New text Segment ==========================\n",
      " when the king was all still  very beautiful , the king  once one , the king  , however , went home  until the king 's son  and the king , and  the king 's son went  to bed , and the  king 's daughter went home  again , and the king  with the king 's son  came and the king 's  daughter went home and went  to the church and the  daughter and the bride ,  and the king 's son  went home and went and  asked her what it was  , the king 's son  was so beautiful and the  king 's daughter went and  put the little which and  her daughter and the bride  was so beautiful and the  bride went and cut her  little foot and the bride  opened the door and had  the bride whom the king  and son went to eat  her , and the king  's son went home and  led him and the king  with the bed and had  not a glass mountain ,  but the king , however  which would not hear it  . the king 's son  how the king also ,  an end of a glass  a glass which might be  an beautiful daughter , and  he went home and cut  him a glass which which  the daughter had no one  , the king 's daughter  , and the king 's  daughter went and put a  great deal which should be  the glass of death .  the man 's son went  back to the bed and  said , `` do you  come and fetch my wife  and you have my wife  . and the king 's  daughter went home and went  to the church and the  bride , and the door  , and at last the  king 's son went and  asked her that the king  also , whom the king  was so beautiful that the  man had only a glass  one of death . the  king 's son went home  with him , the bride  did not hear the whole  castle . and the king  of the end of the  door , and ordered the  king home and put the  whole night and the bride  and the son , and  the son went home and  led him and the little  castle 's daughter , and  at last the king 's  daughter went home and went  so a little man and  the bride went home and  told him the king and  will not be still for  my wife . '' the  man went home and went  to the church and the  others , and the king  's son went home and  led him and the king  and death , and the  king 's son went and  went home and fetch the  church and the bride ,  until the king 's son  allowed him to eat the  castle and the king 's  daughter went home and went  so a little man and  he went and put the  little which and the bride   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 160)\n",
      "(58).(26).(95).(55).(93).(6).(88).(54).(59).(66).\n",
      "Average loss at step 161: 2.808986\n",
      "\tPerplexity at step 161: 16.593086\n",
      "\n",
      "Valid Perplexity: 80.52\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 160 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " who would not believe it ,  for the councillors said ,  this that i will not  go home , but that  it is not so much  the whole night , and  he went home and begged  that the king was so  handsome that that she was  so so so that the  king said , i will  go home home and fetch  my axe , and cut  it a great deal which  money for a great deal  speed . the cat threw  it behind her a glass  or half that the whole  glass which the glass which  he had a great deal  or money . the youth  once happily until the cat  would not know what he  said , i will not  not not be my wife  , and you have no  fear , but that that  he should not be still  like a great distance which  would have my daughter in  it . and the others  said , i will not  not be my bride ,  and will not be still  for a great glass which  which my daughter . ''  the girl said , i  will not go home .  `` this that you can  go home . the girl  said , i will not  be my bride , and  will not be still in  a glass which my wife  whom my axe , and  go home and fetch my  axe , and cut the  blue light which that a  small deal which which should  have , and must not  get home and cut a  great deal which which had  such a great deal which  things for a great glass  or half a great deal  which that it was a  great deal of money .  the youth went home and  went home , however ,  it was a glass of  wood , but as that  as the king was still  warm and the glass which  the children were over ,  the others not as this  a thousand deal , if  you will not be as  it is there . the  girl said , i will  take you home and not  fetch my axe , and  let them go home and  fetch my axe , and  carried him home and fetch  my axe , and cut  her hill which that a  thousand deal which which should  go home and fetch my  axe , and cut the  glass gold which it should  do that it is not  for this . the cat  said , i will not  be my bride , and  must not be still like  it so long , but  he did not know what  , he went home and  said , i will not  take it home and my  child , i will not  go home . the miller  said , i will not  go home and fetch my  axe , and cut the  great glass which which the  glass which you will not  wife that it must be  as that , and had  a great deal of money   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 161)\n",
      "(31).(75).(90).(53).(86).(13).(1).(37).(33).(95).\n",
      "Average loss at step 162: 2.869598\n",
      "\tPerplexity at step 162: 17.629928\n",
      "\n",
      "Valid Perplexity: 74.79\n",
      "\n",
      "Generated Text after epoch 161 ... \n",
      "======================== New text Segment ==========================\n",
      " rich king 's daughter , the  man was so handsome that  that the little tailor came  out a great deal which  which should be set home  and cut him a great  light and the fire .  that did not get it  on it and the little  little horse said , i  will not do you not  my wife . you will  not do it . you  shall not do it for  it . and the king  says , you are you  . you are the queen  for my wife , ''  and now the fox said  she did not know what  it should be as that  . the little tailor was  still in his life and  did not know that it  could not be cut it  and it should be so  so that the little tailor  would not believe it ,  but it should be set  over the little light and  all it was so long  that it should be set  it in its life and  threw him so a great  glass which it was no  wife , but the fox  cried , `` that are  it for you , and  do not know that you  are no one for them  or not , and must  go home and fetch my  wife , but you must  go home again , and  the king says , i  believe that you do not  think that you not are  it . you must be  so beautiful , that that  that i will not not  it you . you are  your wife , '' and  did not know that it  should be set out of  her bed once once in  her life , but the  fox said that it should  go home and said that  that it must be a  handsome man , and the  queen was son that the  king was so beautiful ,  the little tailor , however  that should be cut out  , and as the king  would not believe it ,  and did not know what  it should be as that  for the king , and  let him go home and  put it on and the  queen said , `` that  do not love it ,  '' and did not speak  the thread , the little  horse says , `` that  do you do that that  my father would be be  so wicked , and was  full , and that the  virgin mary and the king  cried , `` you are  your dear son , you  will not be no one  for me , and not  it are no one ,  and not not you that  i will not be my  son , you will not  get home and fetch you  , and that you are  my dear son , you  shall not do you not  my wife . you are  your father , you must  go home again , and  you are no one for  me , and that you  should be so beautiful ,  what that i are so   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 162)\n",
      "(22).(24).(41).(74).(81).(11).(31).(86).(29).(38).\n",
      "Average loss at step 163: 3.029333\n",
      "\tPerplexity at step 163: 20.683428\n",
      "\n",
      "Valid Perplexity: 76.64\n",
      "\n",
      "Generated Text after epoch 162 ... \n",
      "======================== New text Segment ==========================\n",
      " saw it . the bear was  full in a wood and  ran home , but the  princess , who was so  so that the old woman  came home and put it  on the fire , and  children the whole court and  began to play , the  queen , and the king  in the country , the  princess , however , should  have an end of the  country . the king said  , `` that do not  think that i must be  so so so as that  that i do the whole  light . the man and  the whole woman which were  full of straw . the  tailor , however , should  give it up , and  his little woman sat behind  his wife , and the  whole court which was so  handsome that that the last  fire of the country ,  but the little tailor cried  , `` do you do  my human form . ''  that went home and kissed  it and and fire ,  that the king 's son  . the bear was so  beautiful and this that she  would not learn this ,  that the whole court which  she had , and the  king 's daughter , the  last man , and the  king and queen of the  whole court which she was  full of wood , and  that the king 's son  . the king 's daughter  came home and went home  in the country , the  queen , however , should  do not believe that the  old woman was full of  fire . and the tailor  came home and went home  and cut the fire and  bread and the king ,  until the whole house was  bright for church , and  it at last was so  beautiful and that the little  woman were full of wood  that the old woman were  full of wood , and  as the old woman were  so beautiful until the old  woman were full of wood  , and the king ,  how the old woman was  full of wood , and  it went home again and  cried , `` that is  an old woman , and  that you shall be my  daughter . the tailor said  and said , `` that  have you be so so  so as as long as  i do , that the  old woman , and the  king 's daughter , and  the king , who were  a beautiful man , and  the king , and the  king 's son , and  that the whole court which  were so beautiful that the  king , the queen was  a handsome man , and  all the king and queen  , and the old woman  began to cry and that  it should not learn the  little woman , and the  king 's son pleased her  little little man and the  little woman which were still  beautiful until the whole country  . and the king to  last end of the country  which the king , and  the king , who were   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 163)\n",
      "(11).(29).(2).(38).(35).(40).(67).(99).(74).(12).\n",
      "Average loss at step 164: 2.928472\n",
      "\tPerplexity at step 164: 18.699033\n",
      "\n",
      "Valid Perplexity: 80.67\n",
      "\n",
      "Generated Text after epoch 163 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " . and the king said ,  i will not return to  it , you are so  so beautiful that that that  , the king is so  well that you are so  so beautiful that you are  my wife , and you  be so beautiful that you  shall not learn that ,  it is not my wife  . the king said ,  yes , that must not  return to that the whole  court , and will not  learn what you are so  long , that that the  whole court , and will  go home home home and  do that you are so  beautiful that you are so  beautiful that you are not  so well . you are  not so fine that my  husband are not to be  your wife . the king  and that that the king  would not come home and  tell him the whole which  , however , would be  all in a place and  they said , that are  not a fine fine trade  , you , dear father  , the king 's daughter  god , that do you  come home home home and  take you home and die  again . '' the bear  said that the king was  so beautiful that the king  said , i will not  return to you , and  tell you the king and  have the king and do  you that , i will  return to you that i  am so so as that  that you are you to  you and that you do  anything for you . ''  this afterwards the king said  , that is not all  your wife . and the  devil said that the devil  said , i do not  know that it is not  so well . the father  said , i will not  return to you , and  go home home home and  take you home . ''  `` that you do not  know what you do ,  my father ? '' ``  the devil ? '' the  king , however , should  come home home home and  fetch you , and soon  that the devil said ,  that is no father in  a great glass which which  will go home home home  or not . that is  not your wife . the  youth and the king ,  happily the whole country ,  until the king was so  beautiful and all a great  black stone which was so  handsome that the king and  queen and the whole happiness  which had no one life  for them , and the  king , however , should  come home home home and  shall die , and will  go home home home and  fetch you , and that  that that the devil had  come home and cut her  money and the king and  the king 's son ,  the king , however ,  did not be so so  so that the whole court  that , went to the  old king , and will  go home home home and  keep you life and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 164)\n",
      "(58).(2).(73).(55).(36).(78).(80).(41).(46).(3).\n",
      "Average loss at step 165: 3.056833\n",
      "\tPerplexity at step 165: 21.260118\n",
      "\n",
      "Valid Perplexity: 74.59\n",
      "\n",
      "Generated Text after epoch 164 ... \n",
      "======================== New text Segment ==========================\n",
      " the king and the king said  which he had not him  her wife that that the  old woman had to be  still in his bed and  looked at each other and  all each one , and  were all still alive and  said , i will not  not hold that , and  not a little deal of  gold , and the wolf  said , i will not  be my wife and the  man and the king 's  son , and the wolf  said , i will not  do it , you will  go home home home and  take my father and wife  that i will not not  hold that . the wolf  said , i will not  return home , you shall  be my wife , and  how no one can be  all for the king and  see that that is no  father , you are my  bride , and my mother  , and that me as  a great stone and is  no one to be my  daughter . the wolf 's  boy and the king and  all of joy , and  each one them and the  king 's daughter , and  he went home and put  it in his bed but  the wolf said , i  do not know that you  should be still still as  well . the wolf and  his daughter , and the  king , however , will  go home home home and  go home and do my  father , and the wolf  said , i will not  get home and fetch my  axe , and do you  know that i will not  do it , you will  go home home home home  or eat and drink and  have my wife . the  wolf , however , went  for a great deal which  came in his bed and  looked for each other and  said , the wolf 's  son , you are my  son , you will go  home home and fetch my  father , i will not  not hold that , and  said , i will not  not hold that , and  that no one , and  no one , and the  king 's daughter , the  young woman said , that  is not my wife and  be my wife and not  do that , the king  said , i will not  go home and fetch my  wife , and that the  old woman said , i  will not not hold that  . the wolf went home  , and at length the  young king went home and  cut all them and the  bed , and the wolf  said , i will not  go home , but the  youth said , i will  go home home home home  which you shall not learn  you , and that you  shall not be my wife  but that you will not  go home and fetch my  wife , and the wolf  cried , i will not  go home and eat my  wife and eat my life  or not that you ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 165)\n",
      "(29).(6).(59).(93).(67).(46).(5).(44).(11).(4).\n",
      "Average loss at step 166: 2.897650\n",
      "\tPerplexity at step 166: 18.131480\n",
      "\n",
      "Valid Perplexity: 69.73\n",
      "\n",
      "Generated Text after epoch 165 ... \n",
      "======================== New text Segment ==========================\n",
      " time the king 's daughter ,  the king 's daughter ,  but the king said the  king , and not not  do the king , and  that the king 's daughter  and the king 's daughter  and the king 's daughter  and the king 's daughter  came and said that the  faithful john did not know  what the king was so  very long , and at  last the king and the  king went home and said  , i will not be  fat for the king ,  but the king answered the  queen , that i will  give you a stone and  the glass of my life  that i will not be  so so that the king  answered , you are the  king 's daughter . what  this i will not be  as my turn , i  shall not not be my  bride , and will not  go and see that you  will be fat for a  few stone , and the  faithful john said the king  . what do you know  this , i will not  have it , i will  give you a great deal  one for them and see  that you shall be fat  for a great stone which  had be be fat and  cried the king , what  have you come for my  father , i will not  have it . what do  you know what i will  . how do you think  that that you may be  here , but that is  not a fine fellow ,  the king , god is  all my wife . the  girl went home and went  out and cut the glass  glass which the children had  come and the king home  and cut it home and  said that the king 's  daughters were . what do  i know it and not  have to my wife and  see that i will be  my wife , and if  this i do not know  that it is be so  glass that you will be  fat for a stone and  that you will be fat  for a stone , and  cut the hill which it  are so that if the  young king did not know  it and the king 's  children , but the king  said , i will not  have it . what do  not know what i will  until what do i will  not have the king ,  but the king said ,  you will not be fat  . hansel , however ,  said the woman , i  will not not not be  here , we will not  have to my life .  what do you know what  it shall be fat for  a stone and that you  must be fat for the  children , we will not  go home and fetch my  father and this you will  not be fat for the  duck . what shall you  come home and fetch my  life and have the king  in his life and have  the king 's daughter ,  we shall not be fat   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 166)\n",
      "(14).(44).(60).(82).(75).(79).(4).(76).(69).(89).\n",
      "Average loss at step 167: 2.973567\n",
      "\tPerplexity at step 167: 19.561575\n",
      "\n",
      "Valid Perplexity: 78.64\n",
      "\n",
      "Generated Text after epoch 166 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " it , and the child had  fear , and went and  and it at all her  body , and it came  to the bed and cried  the child , and that  at last the child had  rest which the earth the  break that that it was  present in their little bed  and it was all to  do it and do you  for a great deal which  any one pieces . the  king went home and cried  that the child had rest  in the bed and the  children the daughter and it  lay over her life and  go and and the king  and rest beneath the ground  until the child had rest  until the child had rest  and the glass which the  wife had the daughter the  castle and it at last  her child came out and  set it in a bed  until the young man ,  but the king and all  his daughter came and the  king and all the daughter  had it , and it  at last the child had  fear , and went out  so a little man and  which had no rest until  the child had rest which  which the glass of all  such a stone and and  all at last the child  are rest beneath the ground  . the man continued ,  no one , and no  wife had not be so  human so , and that  was all in , and  that at last the child  was rest beneath the ground  , and at last the  child had rest which the  children which it was no  rest until the child had  fear the children , which  the child had rest which  a glass all a horses  for the earth which was  no one in the ground  or eat . it ,  said the child , that  do you come out .  my child , you have  an bride , and the  child had rest which the  forest had rest and a  straw , and how it  began to be hold and  it at last the child  are rest beneath the ground  . the man continued ,  `` that that you are  now no one , that  you have no rest and  bread which your wife are  fat , and that no  wife had no one any  pieces . it must be  as as the child ,  and the child had rest  one and the two children  again and the daughter ,  until it at last the  lord says , i have  the bride you be so  long , and that that  are no one in the  life and have no rest  for them and see that  i have no rest until  my heart . that the  child had rest which the  earth the rest which it  heard until the child was  rest beneath the ground and  cried , and not no  things , but the young  queen came home and put  him and a glass which  it at last the child  are rest beneath the ground   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 167)\n",
      "(3).(98).(11).(85).(16).(12).(54).(37).(0).(23).\n",
      "Average loss at step 168: 3.134012\n",
      "\tPerplexity at step 168: 22.965932\n",
      "\n",
      "Valid Perplexity: 78.49\n",
      "\n",
      "Generated Text after epoch 167 ... \n",
      "======================== New text Segment ==========================\n",
      " have you be fat , and  and the king 's son  was so beautiful that the  king 's son was so  beautiful that the king was  so beautiful that no one  would not be fat ,  which should not be fat  . what will not do  to you . i will  do you that . what  have you do . gretel  , gretel . gretel and  said , i will be  here and will not be  fat . what will not  have you you . gretel  , gretel . gretel ,  where gretel did not do  that it was no one  sleep for her . gretel  was not no one to  be fat . gretel ,  hans . gretel , however  which should do to her  . gretel , however ,  would not be fat ,  the king did not know  how to do her .  what will you give you  for me . gretel the  king 's daughter . gretel  cried her , gretel .  gretel , hans . what  give you no one .  gretel should be fat .  that was so well that  he should be my bride  . what will you do  or not . what 'll  n't do you . and  the king , however ,  would not be fat .  gretel did not know what  in the stable and cut  her life and gretel ,  the king , however ,  did not be fat .  gretel , gretel , and  said , `` what will  my bride do you that  . what do you do  . what do you do  . what do you do  , but the king says  . gretel . gretel ,  mother . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  do . what do you  give it . gretel ,  however , should give you  something . what will you  give me the little little  goat and gretel , gretel  . what do you do  , but the king says  and believe that you should  give you to life .  this this the king did  what no one could be  so beautiful that it should  not be fat , and  that the king was so  white that the king 's  son were so beautiful ,  that the king 's son  was in the bed ,  until as the king was  so warm her a little  one and this the king  would not believe that the  young king is in a   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 168)\n",
      "(15).(26).(29).(46).(80).(9).(42).(32).(22).(34).\n",
      "Average loss at step 169: 3.200729\n",
      "\tPerplexity at step 169: 24.550417\n",
      "\n",
      "Valid Perplexity: 77.20\n",
      "\n",
      "Generated Text after epoch 168 ... \n",
      "======================== New text Segment ==========================\n",
      " an king , and the king  which was not to go  out of the window and  put the little little table  from his life , who  was no one ever be  my wife . the little  one said , what do  what have you here for  my wife , or you  can be my wife and  his life and you are  here , i will not  be my bride in .  '' and hansel went and  put it in the little  stable and and the queen  , and the king and  that she had no one  but a little wife and  her little little house and  and the king 's daughter  in the country . how  do you have my bride  and be my wife and  her little wife and her  father , and the angel  the queen and the king  of the queen which the  queen were all to be  an queen , and the  young bride come home and  be my wife and her  little wife which it are  all my wife and her  life and life , and  not the king 's daughter  to death . the bride  took her little wife and  them , and it was  the bride and the king  's son , and the  young bride come home and  be my wife and her  little wife , and the  king and the queen were  full of joy , she  went home and cut a  new little glass which it  must be my bride to  eat . i will not  be my bride to be  for my wife , and  that is a little man  , that i will go  into my chamber and the  king and death that my  father , and that you  shall be my wife and  my wife and them which  my wife are silver little  wife and my wife do  wife . the man said  to her , i will  not be my bride to  say , i will not  my bride , and that  it is all my wife  , and that is my  bride . the bride and  son and the bride ,  but the bride said ,  what are my dear wife  until my life and may  be my wife and her  wife and them which it  shall be my wife and  her little wife and her  wife , what have you  my wife , whom the  little house whom my wife  shall be my wife .  but the king said ,  what are my wife ,  and my bride , and  go home and see that  i will be my bride  - and that you are  no one , and the  old man has no one  be my wife , and  my child , what have  no one can be my  father , and that you  are my bride , and  you shall be my wife  . and that is my  dear wife and my wife  such a little stone which  i will be my bride   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 169)\n",
      "(84).(79).(12).(69).(94).(70).(32).(44).(49).(42).\n",
      "Average loss at step 170: 2.900685\n",
      "\tPerplexity at step 170: 18.186597\n",
      "\n",
      "Valid Perplexity: 79.63\n",
      "\n",
      "Generated Text after epoch 169 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " , the king and his daughter  to death . and soon  after the king 's daughter  , the man was so  long as if it would  not have an end ,  that the king 's daughter  , and the king ,  and the king , who  would not come home and  keep on the ground and  put a little cloth which  which had no one power  all him , and the  faithful john said , dear  man , i will not  have it for you and  be my bride , and  shall not be my wife  but it and you have  my bride , and you  have no one for me  that you have no one  but my wife , i  am my bride to be  my wife , and the  old man says , i  know that i will not  be my bride , and  will not be my bride  for me , i will  not have the whole ones  my wife in its life  or my wife , and  you will be my bride  . and the man said  , i will not be  no one for me ,  that i know that you  must be my bride to  die . i will not  have it for you and  be my bride , and  can not know what it  shall be my bride and  do , but that you  may not an end ,  you shall be my bride  . the man and the  bride said , i will  do it that i will  be my bride , and  are my bride , and  you shall be my bride  , and shall not be  an bride , and the  king 's daughter , and  that he would not be  an bride and the king  of death , and have  not no one any one  for him , but that  he would not be still  like an end , and  the king 's daughter himself  of the king , who  would not come out ,  until at last the king  said , i will not  be my bride , and  shall not be my wife  but the man and the  king 's daughter , and  the king , who was  just so long as it  would do to eat and  drink , that he will  give you the whole which  life be my wife for  him , i will not  take it home and eat  or drink and have my  daughter for me , i  shall not be my bride  that you have no one  but a stone which is  be my bride , and  can not know what you  come and my wife is  silver and a stone which  is not my wife .  what do you come and  eat and eat and drink  . the man , however  which would not be set  for him , said the  woman , you shall be  my wife and him and  come home and eat and  eat and eat and drink   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 170)\n",
      "(68).(25).(30).(34).(17).(72).(37).(20).(75).(77).\n",
      "Average loss at step 171: 3.003422\n",
      "\tPerplexity at step 171: 20.154396\n",
      "\n",
      "Valid Perplexity: 77.65\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 170 ... \n",
      "======================== New text Segment ==========================\n",
      " the whole house , and as  as as as as if  the man would not have  an bride , `` that  is the whole house and  the whole house , and  the whole man had ever  no one sleep them .  no one knew how to  come home and eat ,  until the whole house which  the girl was full and  a little house her and  all a glass and little  and happiness and the whole  whole happiness , and the  last one day the house  began and eat his life  until the wife would not  know it , and no  peace . the girl went  into the town , and  no one knew how to  stop it . the man  came home and just went  back , and as the  girl saw the whole man  began to eat its life  and the king went home  until the whole house and  a little house and and  his wife the queen came  to the king , and  no one could not return  home , and a handsome  happiness , and the little  little house the little house  until the whole house which  the girl had the bride  come home home home and  sing a little little house  the whole house and the  little house , and no  one could not return to  a little house , and  how it did not know  how to eat it and  the whole house , and  no one knew how to  stop the little bed and  said , `` dear little  god , god have the  wonderful man which i must  return to you . and  it was not long as  it was and the whole  man came home and not  the man , however ,  did not know how to  return home , and the  last man which the bride  were so beautiful and all  marriage and said , ``  what do you return and  be my bride , and  it is for your little  bride , and i have  it . '' and as  that as the girl had  rest and a little house  the little house and the  queen came home and cut  his way and the whole  glass which she went and  eat his wife and her  mother and the bride came  out and the whole house  the little house and and  her wife said , ``  dear little child , god  dear mother , and if  it do my wife wife  , and you have it  and my bride , and  my bride , and it  is for a great deal  of wine and hunger and  or not , how the  little man would not come  to eat and eat a  sack and not that ,  said the girl , and  that it was not long  before the whole house she  began to cry and the  whole house , and the  stones and until the last  king came and the king  in his bed and said  to the king , god   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 171)\n",
      "(30).(52).(4).(5).(22).(72).(26).(35).(63).(68).\n",
      "Average loss at step 172: 3.075309\n",
      "\tPerplexity at step 172: 21.656583\n",
      "\n",
      "Valid Perplexity: 78.22\n",
      "\n",
      "Generated Text after epoch 171 ... \n",
      "======================== New text Segment ==========================\n",
      " corner . the father said ,  no , you will not  see that my father ,  said the father , i  must not know that you  are a great heap which  were a heap which is  silver and a great deal  which which which is a  man in the forest and  be a little bird ,  said the father , and  how the father will not  be my wife . the  father and went home and  believe that he should be  the bride , and the  father said , i will  go home and pass ,  my father , will you  be my wife and a  beautiful man is not a  beautiful bird , that is  your bride , but my  mother i will not not  my bride , and the  bride said , i will  be my bride , and  went home home and eat  . the man 's son  came home and kissed him  he went and put a  stone which his wife life  them and joy , and  the king 's son were  full of joy , and  his father said , i  am my bride , and  what is that , said  he , that is not  my wife . the man  said , i will not  do you , and not  my wife , and the  man said , i will  go home and pass ,  but my son , will  n't be my wife and  that i will not be  an bride , and the  son said , i will  be my bride , and  kissed each other , and  went home and sing that  that the king was so  long that the man was  not to go home and  kissed him , and and  his son went home and  sing that , and the  bride was so that he  would not know what a  stone of the king had  him her wife and his  bride , who was so  beautiful that the father was  standing in his hand ,  but he was not a  beautiful bird , and at  length he went home and  said , i will not  be my bride , and  that you will not be  my bride , and you  must be my wife and  my wife and him and  said , i will not  do you , and not  my bride , and the  huntsman went home and kissed  each other , and went  home and sing that that  they went home and put  a stone which his wife  life his wife and his  face he was not to  eat and drink , but  no one would not be  like an bride , and  when the king were all  him , and he went  in the forest and was  all a great heap which  passed a great heap which  were a heap of straw  . he went home and  kissed him , and and  herself until the father were  full of a great heap  of wine , and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 172)\n",
      "(68).(58).(30).(54).(3).(15).(51).(32).(92).(29).\n",
      "Average loss at step 173: 2.640142\n",
      "\tPerplexity at step 173: 14.015187\n",
      "\n",
      "Valid Perplexity: 80.66\n",
      "\n",
      "Generated Text after epoch 172 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " hunting . the girl was so  long as the old woman  began to cry and the  queen was the old man  for the table , and  looked in until the stones  and which fell on her  life and showed her the  old man and her wife  , and it was a  king 's son , and  the old woman came home  for a little heap ,  ate her horses and which  fell on a tree and  covered them home and drink  for her horses , and  cut a new little ring  which the little ones belong  to king , this i  will not live for me  . the little hare comes  which you will go home  or not , or that  i will be my bride  of death . '' the  old woman went home and  led her so a little  glass , and the little  ones belong to king ,  god will not be my  little bride , and you  are my bride , and  will not not be my  old woman , will you  be my bride , and  says , i will be  an bride '' and the  bride said , `` that  are my bride , or  i will not be my  wife , and you are  my bride , and will  go home home and fetch  my little ones , and  my bride , and the  king 's daughter , who  went and led them so  the whole day and the  king 's son was so  beautiful for a tree ,  that she took the ring  for the ring , and  fell on a tree and  laid her in a great  heap which it was a  handsome man , and went  on the table and the  old man and it was  a heap of wine which  fell on the tree and  said the little ones ,  but the old woman were  the bride , and the  maiden went home and put  a little ring which it  over an end , and  it as the old woman  and one day the old  maiden was a heap which  would come home and eat  for them , and the  old woman were all in  vain , and led her  into a great heap which  one which which the little  bird which was there for  the king , and the  two ones belong to king  , this the bride said  nothing , and not an  king 's daughter , who  knows that it is a  heap of straw , and  led her home and the  king , who were so  handsome , but the old  man went home and led  him so a stones and  stood in the bed and  could not find it and  sleep as long as it  can be my bride .  the girl , however ,  could not be be as  the man . the girl  did not know how to  be my bride , and  how the little woman is  still dead . the girl   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 173)\n",
      "(67).(74).(42).(65).(35).(44).(71).(20).(24).(96).\n",
      "Average loss at step 174: 2.986343\n",
      "\tPerplexity at step 174: 19.813097\n",
      "\n",
      "Valid Perplexity: 78.61\n",
      "\n",
      "Generated Text after epoch 173 ... \n",
      "======================== New text Segment ==========================\n",
      " am you , and will not  hear you , and that  no one are my bride  and will be my wife  . and when the man  heard that she should be  an bride , and the  young bride did not know  how to do you that  the king 's daughter ,  who were so beautiful that  he should be the king  in a great castle which  would come home , the  king , what the old  king did not know one  word which it should be  all . the girl 's  son and the king 's  aged son were all all  and said the whole night  . the king 's son  too the king 's son  , and the king 's  daughters not the king ,  and the king , however  which should be an king  , 's son that the  young king were so beautiful  that the king 's son  came home and asked it  . the king 's daughter  to him , and were  two one head and the  glass which the king was  in a great glass which  which she should break ,  until she should not know  what a little bird were  in a glass which which  it should be be in  until it , and the  king's son and the daughter  at length she was so  beautiful that she should be  an king 's son ,  but the king did the  old man 's son ,  and the king 's daughter  with him that the king  was so beautiful and the  queen was the bride and  the king , who was  glad to sing the little  bird , and at last  they went home and asked  her what a little bird  began , but when the  king went home and the  king 's daughter , and  he went to the castle  that the king 's son  rejoiced the daughter , and  that the king 's daughter  . `` that is no  father and you you are  the bride , and you  shall be my bride ,  until that is there ,  my bride , and you  are my bride , and  are no one in the  trouble and be my bride  . the girl and the  daughter was not just so  that , however , she  cut it and the old  old woman went home and  set it so that it  should be the king 's  daughters . the man ,  too , however , went  in the whole night and  the king 's daughter ,  when the king 's daughter  , the old man were  once so so the king  went home and the king  's son , and the  girl did not know that  , said the king .  and as the king was  sitting in the glass which  which the king 's son  went home and she was  the bride , and the  king and the king 's  daughter to death , and  could not know that ,  but the king went home   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 174)\n",
      "(57).(1).(25).(89).(54).(35).(15).(24).(93).(83).\n",
      "Average loss at step 175: 2.665289\n",
      "\tPerplexity at step 175: 14.372108\n",
      "\n",
      "Valid Perplexity: 83.33\n",
      "\n",
      "Generated Text after epoch 174 ... \n",
      "======================== New text Segment ==========================\n",
      " which would not know that ,  but agreed that the child  had rest which it should  sing the little house and  ate it and all which  he laid it so a  beautiful handkerchief , the child  were all laid in the  church and it at this  the child came home and  hold as that it so  one as it should not  know that , and agreed  that it , said the  mother . '' the mother  gave him all them also  , and the shoe which  no one could rest in  the bed and it so  beautiful until it were so  so that the child came  her wife and said ,  what do you do .  but the mother said ,  stop , '' said the  bride , and it should  rest in the bed or  it . the mother came  in his bed and which  fell in his bed and  laid them in the bed  pieces and which were so  warm that it should rest  one a beautiful little house  , and it went and  and it agreed one and  which it in its bed  until the house and said  that it should be an  bride and which were so  so so that it was  all in its bed and  which at last the child  came all all which the  child heard this , and  it at one step and  red which it should rest  , and it should not  answer it , and as  one one all all which  it at all all all  which the little table were  all in his hand and  cut it speed and it  as not as it should  rest , and the mother  which at last the child  came all all and the  king , however , should  do not know it ,  for the bird and a  glass which it should rest  until the house . the  mother agreed and know this  , but it went and  how the little mother came  to an bed in its  hand and laid them in  its bed and laid it  in the bed and laid  it in the bed and  laid it in the bed  beneath the shoe and ate  her one until the mother  should not be so glass  , and the child came  all the little bed beneath  the earth and said the  mother , and how the  father did not know that  in the country , and  soon one all all laid  it so a bed the  cry , the cat and  a little bed fell on  his bed , and it  at last , however ,  if the child came the  queen and said , ``  yes , wow , wow  that if my wife would  break , and the child  did not know that ,  but it hold the glass  which it for all the  glass which it in the  bed which heard one which  all all which should be  no one in his hand  which should not rest in   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 175)\n",
      "(56).(98).(35).(23).(94).(34).(58).(96).(71).(25).\n",
      "Average loss at step 176: 2.798890\n",
      "\tPerplexity at step 176: 16.426400\n",
      "\n",
      "Valid Perplexity: 80.45\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 175 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " UNK . `` if you not  you have it . the  youth and the son went  and went home home and  brought it home and said  to her , she will  do you not , and  have the old woman ,  if you are my bride  , and will not be  sure , you will not  eat home . '' the  man went home and took  her for a glass which  which she would not be  hold of a great lake  which she will not hear  what that is for a  day , and the old  man is not i. and  fell to her little bed  which she should not be  hold of a great lake  . if the old woman  were all still in his  cellar and cut the glass  gold and it in the  cellar and see what it  would be my wife .  the man and the old  maiden were all still alive  until she would not be  no one , she will  eat home and see what  i will have it ,  or if you will not  be my bride , and  must not be sure to  be my bride , and  will not be my wife  for them . the woman  began to cry , and  at last the child had  come home and see what  it should be an bride  's son . the mother  also at last , however  the child were all still  in the cellar and cut  him and the king and  hans went home and said  to herself , what will  you go home home and  eat , or i will  take it home and eat  or your little table and  a little bird which you  live for a great deal  one for her wife and  draw some money for your  little one . what do  you will not know that  to eat it , and  you will not be my  bride , and will not  hold it for a great  stone which she will not  see what it is not  that , i will not  be my bride , and  are no one , and  has no one power your  face , i will not  hear you . that she  would not be hold as  that she went out a  great deal which should rest  for a glass which the  old woman was all still  beautiful wife for a great  bird which was no one  any one . the man  rejoiced and went home and  let him eat home ,  she did not know what  like the king , and  at length she said ,  hans . what are you  here for my wife ,  but i will not eat  with me . and the  woman said , `` what  that i will not not  be my bride , and  will not be my bride  . the woman went home  again , and went home  home and the old woman  began to cry and the  queen was all in his   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 176)\n",
      "(8).(74).(73).(96).(64).(45).(62).(17).(80).(72).\n",
      "Average loss at step 177: 3.040014\n",
      "\tPerplexity at step 177: 20.905539\n",
      "\n",
      "Valid Perplexity: 78.54\n",
      "\n",
      "Generated Text after epoch 176 ... \n",
      "======================== New text Segment ==========================\n",
      " live for his wife and the  king 's daughter , and  that it should be be  still in his life and  have the bride fall in  a glass which which the  bride would not have it  so that it should be  set up to the door  where the king 's son  , and the prince went  back home and said that  the king 's son would  come , the king 's  daughter , and that the  girl would be so glass  and all a great lake  that it was the bride  where the king 's son  came and said , ``  father , will you be  your bride . '' the  prince went home and told  them everything that it was  the bride , who was  not it to go home  , and agreed that the  prince said , `` that  too that no one have  it be my wife .  you are the bride and  take the stone and the  bread home and be your  wife . and that is  a horse , and not  the king 's son ,  that the fox have the  king and that the king  will not not be my  bride , and will not  go home and see that  your wife live for him  , and will not pass  it , and that you  shall not pass that that  i will not pass the  wild sea , or that  my father would not be  like my bride , and  will be my wife for  him . '' the girl  said to him , ``  that is the bride and  go home and see that  it is be my wife  . you will not be  my bride , and will  not be too for a  great stone and that you  can not be my bride  for you . '' the  girl went home and told  them everything that it was  a glass of bread and  his wife and him which  they would not be too  like a stone and the  king 's son and the  king's daughter went home and  thought that the king 's  son would not have hold  him his life and said  that the king would not  come home , and the  prince said , `` that  are the bride , and  you will not pass it  or my wife , i  shall not not be my  father , and that you  can be my bride .  '' the girl went home  so that the king was  there of all the whole  day that the king was  all in his life and  should not hold the glass  pieces and said , ``  how have you come and  have my bride , and  that my own life .  '' the girl went home  so that the king ,  however , would not be  like an bride , and  the prince went home and  told him that the king  would not come home and  see the old princess ,  he was the boy ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 177)\n",
      "(86).(23).(71).(62).(18).(20).(3).(68).(85).(32).\n",
      "Average loss at step 178: 2.722950\n",
      "\tPerplexity at step 178: 15.225173\n",
      "\n",
      "Valid Perplexity: 80.17\n",
      "\n",
      "Generated Text after epoch 177 ... \n",
      "======================== New text Segment ==========================\n",
      " , but the king said ,  i will not do it  and be my wife .  how i have it .  the little dwarf said ,  `` that is a little  bird for her one which  i will be my wife  for him . the man  , however , must go  into the bed and the  maiden and the king 's  young daughter . what a  little thing do you have  a glass of wine and  drink , and said ,  do you do that that  i will be my wife  for me . that is  no one that my wife  can not come home and  eat , my wife is  all my wife , and  not that i will be  an king 's daughter and  life and be my wife  . what that have you  will be my bride .  it , however , will  go home and fetch my  father , you will not  be too too for a  new one for me ,  i will do you or  drink , and that you  can be an bride ,  dear little ones , the  little ones belong to wife  . that that the bride  would not come home ,  that the king , who  should come and pass and  the little glass which the  bride said , `` that  are my bride , and  shall be still too good  , and i will not  give you my life and  be my wife , and  that you are my wife  , and that you are  my bride , and will  go home and fetch my  wife , you will not  do that that you are  my bride , and will  give you for my wife  the king 's daughter .  that that i will not  be too still as long  a glass as that for  my wife , that that  will be my bride ,  until you will be my  wife for him . the  little dwarf said , that  is the bride , which  i will go home and  see that my wife are  not an bride , and  my son , that is  still too long that i  are my bride , and  shall not be still too  long . that that is  my bride , and that  i will not be too  still as long as you  do . that is a  large one , you will  do you that . that  a little bird in the  forest , that the little  dwarf had come to pass  , and the king 's  councillors children , that that  that he should be still  glass . the man and  the king and death was  all there . the mother  and the king 's son  came home and went home  for her wife which the  maiden had rest which it  made a little dress and  all a little glass which  he said that the little  brother is be too as  if it are my bride  . the king and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 178)\n",
      "(75).(86).(74).(2).(12).(11).(68).(70).(16).(58).\n",
      "Average loss at step 179: 2.840268\n",
      "\tPerplexity at step 179: 17.120353\n",
      "\n",
      "Valid Perplexity: 80.03\n",
      "\n",
      "Generated Text after epoch 178 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " much good day , what that  do , i am my  little wife , that the  old man did not know  each other . the bear  the king and him and  how the wife am no  one but the old man  , who will be an  bride and wife and this  four glass which the wife  said the little little man  , what will you be  my wife for them or  it . the bear said  nothing that she was so  beautiful that it was not  still there for a little  little glass which the wife  did not know what that  would have hold it so  a little little deal which  that it was no one  anything but it have not  hold for them . the  youth began to cry and  the king and him and  meet for a little which  which was a little heap  one one dress began to  sing that that said ,  i am my wife .  they likewise only hold the  little one each one them  for it . the man  and the old man always  always however and only like  a old man and him  so a little man began  her cry and the king  and him and the old  old man began to cry  . the man took him  and the old man began  and cry and the wife  and him and the wife  said that the little man  began to cry and the  old man 's son and  the old woman was not  very very happy . the  old man began to cry  and the king , however  , should not hold that  it , she took the  little little bed and the  old man , however ,  did not know what the  child , who did not  know what that did not  come and eat it .  it are not there to  do . what have you  come for your little table  my little table , and  the king and him and  daughter , how you are  my bride and eat and  fetch my little table ,  or little table , and  little old man began to  cry and the king ,  who likewise only hold that  for a little table ,  the little man began to  cry and the king and  her wife began to cry  until the old man and  not for an end which  she did not be hold  that the king 's daughter  whom the old man were  sitting there , said the  old man , you have  hold the little which the  little table which the little  woman began to cry .  the king 's son how  the old man did not  come home and eat and  sleep of it . the  man took him all his  food and a little dress  began for a little glass  that that the little man  saw the old man in  his bed and looked for  each one them for his  wife which it should be  like a little and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 179)\n",
      "(18).(40).(72).(99).(7).(87).(32).(13).(37).(98).\n",
      "Average loss at step 180: 2.944873\n",
      "\tPerplexity at step 180: 19.008243\n",
      "\n",
      "Valid Perplexity: 73.50\n",
      "\n",
      "Generated Text after epoch 179 ... \n",
      "======================== New text Segment ==========================\n",
      " the king , however , should  not know that it was  only that she should not  do it . the youth  did not know what the  queen did not know how  that she would not be  be too for until she  did not know what the  queen did not know what  she should be for them  for the tree , and  that the old woman took  it into her hand ,  until the sun was so  very long , and went  to the tree , and  told him that , he  did not know what the  king 's daughter , that  she was no one for  them . the youth ,  after this , the sun  came to the king and  told her not that ,  he would not speak and  let him be like the  tree , and went and  set him into a great  glass which which was a  great glass of happiness .  it went home and led  him home and led her  home and put a great  fish which was a little  fellow in the world ,  she should not know that  it should be as that  . she knows that that  will you be my wife  . the youth went home  so that the sun had  a beautiful ones belong to  what the little man had  not come to light .  but when the king saw  her , and went home  and told him that the  queen was not that ,  went into the forest ,  until she went to an  old woman 's son ,  she went to her great  bed and told her that  he should not have no  human any one . the  youth did not know how  , however , will be  too little and the whole  ones belong to cry .  it did not know what  no one should be my  bride . and when the  old woman saw that the  king was so beautiful that  she should not know what  the king 's daughter ,  but he would not be  still too that , she  did not know how to  speak that the sun were  very long as that is  very long . the king  would not know what the  old woman did not know  that the queen were the  king , and that it  would be an king 's  life and the king 's  young daughter . what do  n't you be my wife  and life , we will  not do it , and  that it is not my  bride , and will not  learn that that it is  not my wife . what  will you do for you  for my wife . ''  the next morning the young  folks said that she would  come to light . the  wicked woman , however ,  did not know how to  eat her life and an  king 's daughter , and  that she would be still  glass as long as it  should be for a glass  stone and the tree was   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 180)\n",
      "(2).(44).(26).(33).(0).(9).(38).(5).(66).(63).\n",
      "Average loss at step 181: 3.202568\n",
      "\tPerplexity at step 181: 24.595614\n",
      "\n",
      "Valid Perplexity: 74.98\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 180 ... \n",
      "======================== New text Segment ==========================\n",
      " way , said the peasant ,  you are and than you  do . you must be  my bride , and will  go home home and eat  or not . what will  you go out , and  that you will not be  still for a great deal  which money and my wife  any one to be be  for a short time ,  or that you will be  my bride , and will  not be my wife and  do that that you are  there , and not not  as you must be ,  until i will not be  my bride , and will  take you home home and  be my wife . the  king , however , should  do not know that it  was so beautiful that the  father said , you are  not for a great deal  pieces . what do you  get out , and if  i will go home and  go home and see what  my husband , i must  not be my wife ,  that i will not be  your bride , and you  shall not be my wife  , and will not be  my bride , and will  take you home home home  or not . what is  my husband , and that  you will not be my  bride , and you will  be my wife , and  will not go home and  be my wife . the  king , however , will  be my bride . and  he went home and kissed  his wife and said ,  i will not do it  . what will you go  out , and that you  will not be my bride  , and you will not  be my wife , and  do not know what you  shall be my wife .  what is the father ,  or i will not be  my wife , and will  take you my wife and  you will not be my  bride , and you will  go out and see what  my husband , i must  not be my wife ,  but i will not be  here , i will not  be my wife , and  went home and kissed him  until the wood . the  king said , i will  be my wife , and  will not go home and  see what you are so  and that you will be  so well . what is  you here for my wife  . the king said ,  i will not do it  , i will not be  here , you will not  go out , and will  bring you home and take  your wife , i will  not be my bride ,  but you will be my  son , you will not  be my bride , and  you will be my wife  . the man and the  son said , you are  the bride , and the  jew , who is a  fine beautiful wife , and  i will not do you  that that i will not  go home , and the  king said , i will   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 181)\n",
      "(48).(13).(46).(70).(18).(29).(33).(44).(17).(89).\n",
      "Average loss at step 182: 2.762169\n",
      "\tPerplexity at step 182: 15.834154\n",
      "\n",
      "Valid Perplexity: 80.19\n",
      "\n",
      "Generated Text after epoch 181 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " a great glass of straw .  for her wife had the  wife it for her little  child , and the little  girl would be present as  the gold , and the  little dwarf said , i  am here , or i  will be my bride ,  until the mother says ,  i will be my bride  the child . you will  take me out , and  then that as it shall  be my bride , and  what are you here for  my wife , and you  do not know what that  are you for it ,  or you shall be still  too that that i do  you for me , and  it is not here ,  i will go home ,  for the arm always ,  until that was all to  the reward , and the  arm always came home and  drink and if she would  do it for it .  the girl went home and  kissed her little bed until  it was all to rest  all the earth that the  girl had rest which it  had no rest until she  did not come . it  agreed that , i will  go home , or you  can be my bride ,  for it should be ''  and the girl went home  again , the girl went  into her bed and cried  to her , i will  be my bride , and  if you do not know  this that i will be  here , and that i  must be my bride ,  or i will be my  bride , and how no  father would not have the  money . the girl did  take that it is not  in the morning , and  cried , i will go  home and go home ,  for the arm always ,  until as she will be  present as that that have  it be for my life  , and that this that  the child had rest which  a glass all all an  great straw , and when  she had an bride 's  wife and that she would  go home . the last  said , i will go  home and be my wife  , and it is no  human all . and the  faithful dwarf held the arm  home , and as it  was all so so that  it was be present for  it and cried , and  a little cloth which it  had no rest until the  shape which the child had  rest which it should be  come . then the little  girl held it so that  that it as no one  could be my rest and  a glass pieces , and  as it was all so  good that it should be  '' at all the earth  , and the girl held  it so a glass which  she went home and set  him so that the whole  day which she was so  straw . that all a  glass the straw which it  agreed that that it was  rest for a glass which  it was all to no   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 182)\n",
      "(97).(82).(9).(91).(64).(55).(0).(22).(79).(65).\n",
      "Average loss at step 183: 3.138710\n",
      "\tPerplexity at step 183: 23.074084\n",
      "\n",
      "Valid Perplexity: 74.19\n",
      "\n",
      "Generated Text after epoch 182 ... \n",
      "======================== New text Segment ==========================\n",
      " spin . and at last the  mother said , i will  be my bride , and  shall not be my bride  but the child , who  are you . and the  messenger went home and asked  the king what the bride  answered , i will not  be my bride and the  child , and not not  know it , and went  out and see that it  had not be my bride  , and when it was  an old girl , who  pleased her child , and  how as it is no  one , and the aged  girl had not be be  so long as it should  have the bride with her  and the king , who  was so beautiful that it  would have an king come  , and the king 's  aged daughter . the child  said , i have no  rest which it may be  my wife until her life  . the child said ,  what have you not my  bride , and that is  the true bride , my  child , you have no  wife , it i will  be my bride , and  as a little girl had  rest which the king said  and will not not have  your wife , and that  i will not be my  king , and you will  not be my wife but  it , i will not  be my bride and the  waiting-maid . the paddock was  there , and that a  couple of horses , and  did not know that it  would be my wife but  it . the girl did  rest until it should be  present for the wall .  and when the aged ravens  would not be be like  the bride , and the  child said , `` i  will not be my bride  until the aged ravens are  home . the paddock says  , i will not be  my bride and the aged  girl , which have you  have be my wife .  or i have no wife  rest and the king and  that i will not be  an bride '' and the  mother and went home and  said that it should be  an king 's son and  a little girl had not  come home and drink and  all the child , and  if the child had rest  which the aged king did  that it was a king  's son , and the  king said , i will  not be my bride and  one of them and have  not my wife . the  mother said , i have  the bride you , and  how my wife have not  an bride , and the  king , however , should  come home home home and  i will not be my  daughter . the paddock says  the aged woman , ``  my child , what will  you be my bride to  an human hands . my  bride says , i have  my wife come out and  my wife the king and  the king said , i  am my bride and the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 183)\n",
      "(36).(3).(17).(9).(21).(64).(58).(33).(5).(32).\n",
      "Average loss at step 184: 2.666416\n",
      "\tPerplexity at step 184: 14.388304\n",
      "\n",
      "Valid Perplexity: 75.78\n",
      "\n",
      "Generated Text after epoch 183 ... \n",
      "======================== New text Segment ==========================\n",
      " him . the man went home  so that he did not  lose the life and should  go home and fetch his  wife , and the father  said , i will not  be my bride , and  said , i will go  home and eat and do  that , i will not  not do it , answered  a little time , and  what is the child ,  break a new new new  coat , and that that  that that the little ones  belong not cry and do  a little ones to be  my wife . the man  began to cry . the  man 's daughter and the  old man 's daughter with  the king , which the  little ones belong to wife  that the little ones belong  , and said , i  will be my bride ,  but if you do not  think that you not as  little as you can not  any one for me .  that you do not know  that i will be my  bride , and said ,  what is the little ones  for me , you will  give me the little ones  for me and do you  think that that the little  ones belong to cock ,  that i will not be  as you do so that  you shall do it for  your wife , and you  shall have the little ones  fall on his bed and  asked the old woman that  it is no one for  me , you have no  rest and it my wife  love me , and the  little ones belong to wife  and you shall be still  dead . the girl did  give him the little ones  belong to little one ,  the king 's daughter .  what are you like a  girl . you will go  into the chamber and the  place and the king and  the king 's daughter always  set him home and said  , i will be still  tall a little one for  him , that i will  be my bride , and  answered , i will not  be my bride , and  let me go home and  see that you will be  my wife and and the  little ones belong to king  and this i will not  be my wife , and  that i will be still  tall the little bed ,  but the fox said that  the little ones belong home  . the jew , who  pleased the fox and the  man and the old one  belong to cry . the  father said , what do  you do . you are  the true one , you  are my bride and do  you if you will be  so tall that i will  go home and fetch the  new little dog , and  you will be my wife  home and what are you  for my wife , and  if you will be still  money . what i will  so that you can not  be my wife . and  he went home and led  himself home and put the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 184)\n",
      "(92).(57).(80).(6).(34).(7).(21).(87).(8).(63).\n",
      "Average loss at step 185: 2.976199\n",
      "\tPerplexity at step 185: 19.613128\n",
      "\n",
      "Valid Perplexity: 74.96\n",
      "\n",
      "Generated Text after epoch 184 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " guess , i must not bring  it to light . and  when the king 's son  went and went home and  led him so a great  garments and covered her for  a great stone which was  a handsome dove , and  as the old woman was  sitting in his bed and  cried , what is the  dear wife . the king  once happily until the old  man were just as that  lay there in his bed  or bring it home and  be my wife . the  man said , i will  take my life home and  bring you home and bring  me home and do you  do my life . the  witch and the son the  old woman went home and  went home and kissed her  little wife , and the  young man did not know  how to go home and  kissed him and and carried  it home and made a  handsome man , and the  bride said , i must  like my wife . the  man said , i will  be my bride , and  will not be my wife  . the king , however  that should go out and  fetch his life and said  and you must be my  human shape . and the  great stone let him be  like a handsome which is  just so long that you  must be my wife .  the man said , i  must not see you and  be my wife . the  girl said , what is  my wife , and i  will not do it .  how do you do my  human shape . and the  king and death and the  bride said , i will  take my life home and  must bring you home and  see that you are so  wicked one , and i  am my wife , and  if you do not know  that to go away and  cried , you , i  must not know that to  be my wife . the  man said , i will  be my wife . and  it must not be my  human shape . and the  little witch said that the  king said , i will  take my life home and  will bring my wife to  do , and will not  know that to be my  human shape . and the  maiden said , i will  be my bride , and  if you know that you  will not be my wife  , and i will not  do it . you will  take your life home and  fetch a stone and the  man and the king 's  swept full and happiness and  that it he went home  but the king 's son  went home and went and  led him home and made  an old woman which he  fell in his bed and  said , you will be  all your life and you  have my wife . and  he went home and kissed  him and and happily and  which it should not be  like my wife . the  man said , i will   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 185)\n",
      "(81).(20).(59).(79).(13).(87).(57).(54).(76).(84).\n",
      "Average loss at step 186: 2.737202\n",
      "\tPerplexity at step 186: 15.443718\n",
      "\n",
      "Valid Perplexity: 73.50\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 185 ... \n",
      "======================== New text Segment ==========================\n",
      " one , and the king 's  daughter , and he went  to the king 's daughter  . the cat 's son  came home and went home  to their father , and  the king 's daughter was  so beautiful that the king  were so beautiful that the  old king was there of  the king 's son were  down , and at last  the king 's daughter was  led to the king which  the king were so beautiful  that the king was so  bright that that should not  come home , the king  's son was so beautiful  that the young king was  there to the king ,  the king 's daughter was  a son , and he  was so beautiful that the  old king was still sitting  , and its treasures home  until the king 's daughter  , the king was so  beautiful that the king was  laid to rest which the  queen was so beautiful that  the king had the children  was in his hand and  cried , `` lord king  . `` yes , ''  said the king , and  he went to the king  's son and the king  's daughter , and the  young king were there ,  until at last the king  's son were there ,  the king 's son were  there , and the king  's son were there to  bring it to light and  the king 's daughter ,  and the king 's son  went home and went and  carried him home and crying  that he should be still  still of his life .  but the king 's son  married the young king ,  happily the king 's son  knew that the king was  the young king , and  brought her all and and  her son were his wife  , who were so beautiful  that it should be set  up home and cut his  hand their pieces and laid  them in a glass which  he should come home and  said the young king and  that would have hold the  great pieces , and the  sun were so beautiful that  the king said , ``  if we will be my  bride , and will not  have you . '' the  king and the king was  down that the king was  just so beautiful that the  young king was so beautiful  a glass as that the  king was so beautiful that  that the king was so  long as the young king  was so beautiful that the  sun was so beautiful that  the young king was so  many that it should be  an king 's daughter ,  and the king 's son  should not hold that ,  that the young king were  just as the king 's  lives the king was so  feast , and were down  , and at last the  king 's daughter , and  the king 's daughter was  satisfied and went home and  lifted her her and and  it at last the king  with his life and said  and will not be my   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 186)\n",
      "(46).(74).(84).(1).(20).(9).(44).(36).(75).(34).\n",
      "Average loss at step 187: 2.813282\n",
      "\tPerplexity at step 187: 16.664525\n",
      "\n",
      "Valid Perplexity: 73.87\n",
      "\n",
      "Generated Text after epoch 186 ... \n",
      "======================== New text Segment ==========================\n",
      " then the little dog which had  come out of the house  . the cat , however  which should be no one  but the bride , and  let him go home and  cut out pieces and the  glass of all which he  opened the window and crying  , that the little bride  says , you will be  no one for me .  my child , what have  your wife in my bed  and do not know that  it , be my wife  until my wife , and  i will be my bride  , and will not be  no one for your life  until i will be my  bride . and the girl  was so beautiful that the  bridegroom said , i will  not be my bride ,  until that may the bride  that the bride would not  come and pay for my  little dog . and the  man looked for the window  , and the bride said  , `` i will not  be my bride , and  what may the bride be  my wife . and that  will you be my bride  , or you are no  rest until it i have  my bride . the cat  began the cry , and  as the bride looked so  beautiful that she was not  so so as it was  so long that it was  rest , that it was  all on her bed and  looked on her bed and  ran home and cut her  life and joy that she  could not come home .  she went home and cut  it home and asked the  young bride , and each  one them and the bride  came home and kept her  life and pieces , the  bride was so beautiful that  no one should be be  still pieces . the girl  had no one ever be  my wife . the man  's daughter , however ,  would not come home and  asked that it was not  only no one for her  little life , and the  cat began to sing a  man and was so beautiful  the house , and at  length he asked her that  she was so beautiful that  no one should be be  like a bride , and  this one came full pieces  until the bride , who  had no rest until the  queen was the bride and  the bride , who was  all there , until the  poor man were laid to  pieces on his bed a  little man and it should  come home and be so  so as it was and  that she should be be  like a bride , and  no one might be be  like a little man ,  it did not know what  that no one could be  be like a bride ,  or that that she would  be my bride . the  bride , however , went  to an old man and  carried him home and cut  the gold gold and the  three man 's daughter ,  however , was all to  an human home . the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 187)\n",
      "(12).(56).(79).(6).(46).(74).(77).(63).(75).(39).\n",
      "Average loss at step 188: 2.719731\n",
      "\tPerplexity at step 188: 15.176237\n",
      "\n",
      "Valid Perplexity: 76.05\n",
      "\n",
      "Generated Text after epoch 187 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " covered him home and said ,  '' dear brother , dear  father and that you will  not be my bride ,  for the fox , however  . said that the old  king was there in the  shape which it should be  be there . and when  the old witch were so  long until the old witch  had to play , and  the old witch saw that  a little man had become  a glass . the old  girl , however , were  so beautiful that the old  old witch were full which  was all there and a  duck for her and the  old man was there in  its bed , and it  hold as that as the  man said , i will  receive it so long that  it which i will not  die for you , and  not not be my bride  , and it was a  bride , and it was  all there and the whole  forest , and the old  devil was so beautiful that  the old man were so  only that they would not  know it , and went  into her bed and saw  it so that the old  witch saw it there and  no one knew how the  children come home and be  the bride , said the  king , and the old  man are now no one  for them , and will  not be my bride ,  or you will not be  receive , you have no  peace , and will not  receive the wife and the  bride , however , will  you be my wife .  the man , however ,  did not know it home  home to her , and  that the three man 's  swept in the forest the  man and the old man  , and went home and  kissed her one white one  its little treasures home her  cry home . the king  , however , went home  until the cat , the  king said , what have  we not be my wife  , or i will not  meet you my wife to  any one , or you  must be my wife .  it was the devil to  do the old woman ,  the old man were so  only that the old witch  was full of wine ,  when the old witch were  full of joy , and  the three eldest whom they  would not receive it ,  so that it was a  handsome man , and if  no one might be be  here , if you will  go home home and fetch  my wife , and the  bridegroom have you . ''  and the cook said ,  `` that do you do  that you will not be  be my bride , or  you will not be now  that you will not be  no one any one to  be my bride , and  i will not be my  bride , or that it  must be my wife to  return home , and the  cook said , what do  my bride , who your   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 188)\n",
      "(73).(53).(20).(63).(85).(46).(27).(12).(70).(48).\n",
      "Average loss at step 189: 2.913919\n",
      "\tPerplexity at step 189: 18.428882\n",
      "\n",
      "Valid Perplexity: 74.66\n",
      "\n",
      "Generated Text after epoch 188 ... \n",
      "======================== New text Segment ==========================\n",
      " for a great deal time to  it . the king 's  swept that the king was  so one until the king  had brought it so a  little little man and the  young king was so beautiful  that the king was so  delighted and kissed her little  beautiful horses , and were  one of them and said  the king . what have  i do that the king  , will be my wife  . the king 's son  will not meet my bride  . the king 's son  seized the king , which  laid for the two man  who was laid in the  door and laid him so  great pieces which were so  beautiful that the king had  no rest but the king  which had come home and  told him that the king  agreed that , the king  could not come home and  told him that the king  were so beautiful that the  king was so beautiful that  she should be like a  beautiful ring . the king  with the two eldest eldest  ordered the ring home to  it , and laid it  there for a glass straw  , but that was so  so that the little maiden  rejoiced at length the king  , who fell on the  ground and went home and  told him that the king  agreed that and if the  king would not come home  . the wolf , who  fell on her little bed  , and the two others  which had brought to light  . the witch was laid  on her little death ,  the king said , i  will not be my wife  for him . the king  continued , you shall be  here , and will you  be my wife . the  witch , and the king  went home and kissed her  little little than their little  bed , which fell in  her little house . the  next morning the king took  a little ring and which  at last the king 's  brothers rejoiced at length the  maiden took a little ring  and which fell in the  fire of the little little  dog and the king with  their great pieces which should  have which it must be  still alive and will be  so beautiful that the king  will not be still in  it . the king 's  son went home and kissed  him , and at length  fell home and herself on  its bed and laid the  three animals 's daughter ,  who fell in a great  forest . the wolf took  her and the king 's  daughter with her that the  young king was so beautiful  one and which was so  long that the king was  the son , went home  with her , and their  one white horses which was  present for a great deal  , the king said ,  i will not meet my  bride , and will be  so beautiful that the king  shall be still alive and  bring it to my wife  which are not for my  wife . the king 's   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 189)\n",
      "(56).(76).(55).(72).(68).(14).(73).(35).(42).(98).\n",
      "Average loss at step 190: 3.074954\n",
      "\tPerplexity at step 190: 21.648897\n",
      "\n",
      "Valid Perplexity: 75.18\n",
      "\n",
      "Generated Text after epoch 189 ... \n",
      "======================== New text Segment ==========================\n",
      " you . '' and he went  to her and went and  laid him on his bed  that was so beautiful and  fetched a little deal man  which had come out for  it , and ordered her  to get out of the  door and the king 's  ears , and went home  and went out and put  him in the bed ,  until the second said the  bear , `` what will  i give you for my  father that my wife are  wife for him , ''  but if you will not  be my bride , and  that you shall be my  bride , and how do  i do that that you  will not be my wife  for them and do not  want you and do you  want , '' and the  king 's son . ''  that he was so beautiful  until the whole night which  fell in her bed and  went home and told him  how the old woman was  just so beautiful until it  over the tree , and  when the king were so  beautiful that the king was  just as the king was  the king , however ,  would not believe that ,  went to her little house  , and the king 's  son went home and went  home and told him that  she would not come and  see that it should be  present for the tree ,  and if you do not  come and be my wife  for me , and that  if you have it like  my wife , the man  replied , `` i will  be my wife to be  a little bird , ''  answered the woman , ``  how do you do .  what shall you give you  all a great deal pieces  . my father , will  n't be my wife ,  or you will not be  like so long , ''  cried the tree , and  he went home and told  her that that she should  have the little dog .  and when the man had  laid her in her bed  , and she went out  and told him that the  little tailor was to be  so beautiful that the king  's son was so beautiful  and night that the king  would not come home and  put out a great stone  which ago , and she  went home and told her  everything that the king ,  where will you give you  for my wife , and  i will not give you  that . '' and he  went home and told him  everything , however , went  to her and went and  went home and told her  what the king , who  were just as no one  ever have it like the  end , and now the  fox said that the king  , however , should not  give him a great deal  one for them and do  not want it for me  . '' and the king  allowed himself in her wife  and the king 's daughter  out of the whole country   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 190)\n",
      "(45).(49).(13).(92).(38).(58).(83).(25).(32).(9).\n",
      "Average loss at step 191: 2.815159\n",
      "\tPerplexity at step 191: 16.695828\n",
      "\n",
      "Valid Perplexity: 76.78\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 190 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " of his life , and the  old woman said that the  old man would not come  out of the palace .  she went home and cut  him a little new little  piece which was so beautiful  that the little men would  come and hold that a  great ones belong to wife  . the man went home  with the king , who  fell home and hold a  great joy , and the  old woman said , what  shall i do , and  that i will be my  son . the little tailor  ran out and cut it  its little light and a  great deal which which had  no wife to eat and  drink , the king was  no one for her wife  but the king , and  that the king was so  beautiful that the king was  just as the king ,  but the old woman was  full , and the king  and death for her life  , and the king 's  young daughter . what do  my child , that do  you know that , we  will not be my wife  . the man , however  which should be hold of  some little which it should  go and pass that it  would be my wife for  it , for the king  , where the king which  to the king , and  that the king 's son  came and the king 's  young daughter . what do  we do for my wife  , and you will be  no one for me for  it , or i are  my wife , and the  little ones belong to wife  that it will be so  very beautiful , i am  my wife , and the  king said , what may  my wife do no wife  and not my wife ,  and the king 's daughter  . the girl was so  only that the king was  no one for her wife  until the old woman said  the little ones , and  the king , who was  just as the king 's  husband , and the king  of death , and the  little ones belong to wife  . the man went home  with her , the old  witch came home and cut  out of the wood which  would be no one in  her life and said ,  my child , what will  we go out and see  what my wife are not  not no one , and  that i will go home  to my wife , and  he said , what may  my wife do no wife  . the man said ,  my child , what will  we go out , or  what may the king says  , what that is ,  or i will be so  long as it should do  a little ones , and  it will be my wife  . the girl , however  , should be no one  . the girl was so  very beautiful , she went  the little bed and said  the old woman , and  went home and cut her   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 191)\n",
      "(4).(77).(98).(97).(22).(23).(34).(84).(93).(67).\n",
      "Average loss at step 192: 3.004952\n",
      "\tPerplexity at step 192: 20.185254\n",
      "\n",
      "Valid Perplexity: 77.14\n",
      "\n",
      "Generated Text after epoch 191 ... \n",
      "======================== New text Segment ==========================\n",
      " ugly which , who was so  glass that the king was  full of joy , and  the king said that he  was no one , and  said that they should be  laid in the ground and  said that he was the  old man , and at  last that was no one  death for this day the  little ones belong to wife  . the man went home  and kissed him , and  fell in a stone and  one horses and which fell  in a stone which was  so beautiful that the king  fell in a great stone  , which was in his  room and cut it home  and the king was so  long that the king was  once so so that they  fell so that the king  at last went home and  asked how the king ,  went home and took the  ring and went home and  went home and said that  he should be a little  fellow in his bed ,  cried the king , and  that the king 's daughter  fell home and the king  , and he went home  and cut out a glass  until the two others and  that so that that the  old woman came and the  king 's daughter , the  old woman was so so  so so so so so  so so so so so  so so so so so  for that they only that  his father had laid them  in the ground and kissed  their horses and laid in  his bed and put on  prison and said that the  old woman said the old  woman , and went and  said the king and the  little house the king said  the child , and that  that the king had no  money . the king went  to the king 's daughter  that it , and he  said that he should be  the king 's daughter ,  but he went home and  asked that the king was  so beautiful that the king  came and the king 's  daughter , however , was  so so so that they  were all laid in the  sea , and that the  old woman went home and  asked it and the king  's son he was in  a great deal which should  be so glass as a  little man and the two  brothers which were laid in  her bed and said that  it was a little house  which he should be so  one that the king was  just as the king 's  daughter , and the king  came and put it on  the ground . the king  said that the king was  just so so so that  she was not so so  as that that they were  not like the bride ,  said the king . what  do you do . that  it should be set off  the little bed and said  that no one should be  so glass that it was  all the king and death  that the king and how  no one might be be  so wicked that the king   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 192)\n",
      "(88).(91).(84).(93).(44).(10).(79).(3).(7).(8).\n",
      "Average loss at step 193: 2.817373\n",
      "\tPerplexity at step 193: 16.732843\n",
      "\n",
      "Valid Perplexity: 77.34\n",
      "\n",
      "Generated Text after epoch 192 ... \n",
      "======================== New text Segment ==========================\n",
      " he went home and cut out  that the king 's son  came and went out and  put the little home and  pass and all fell in  his bed . the next  night the king 's son  pleased her little wife ,  until the king 's son  pleased her , and at  night the king 's son  will not hold that the  child had the bride the  king 's son , and  it was not so long  that the king 's son  had no one sleep and  pass that the king 's  ears , and told her  that the king 's son  came home and cut out  that the king 's son  came and went out and  put the little home and  pass and all fell on  his bed and said ,  what do my wife wife  until your life the king  and the king 's son  ought to see my wife  and the king 's son  , and the king 's  daughters were for all the  shape of a wood ,  until as the king 's  queen went and the king  at where the king 's  mother , who was so  strange that the king was  all laid her head home  again to her . the  woman began to walk and  move so that the king  's son went and told  him that the king 's  daughters were for all that  they will be my bride  . the king 's son  went home and went to  them and asked the king  where the king , how  that the king 's son  should not come and cut  a hair which it should  break home and do that  your wife should not have  it for my wife .  she ought not so so  long as it should be  like my wife . the  old woman went home and  go and see that it  would be my wife and  them . the girl was  there on the ground ,  and the king 's son  went home and went home  and told her how the  child had not come and  see that the king 's  ears , and went home  , went out and asked  her not that that the  girl had to come and  do not know how the  young king will come home  , but the king 's  daughter , however , should  come and drink and the  king 's son went home  and cut out of her  little fortune which had come  but the king 's son  was to see that the  old woman had to be  so so so that the  king 's son was so  glass that the queen was  all until the king 's  mother , went home and  asked , how do you  live for my wife .  but the child said ,  why do you come and  see that my wife is  little wife , my dear  sister , will you be  you for my wife .  the king 's son went  out and cut out its   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 193)\n",
      "(26).(37).(49).(92).(14).(38).(66).(60).(41).(59).\n",
      "Average loss at step 194: 3.126032\n",
      "\tPerplexity at step 194: 22.783407\n",
      "\n",
      "Valid Perplexity: 82.88\n",
      "\n",
      "Generated Text after epoch 193 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " 's children , and went home  and kissed him , and  ran for her so long  until the old woman went  home and cut her a  thousand glass its glass which  that the little tailor ,  however , as she was  all in a great glass  and half a glass ,  until the girl went home  so a great glass of  half a glass which the  children saw a great glass  which that it at last  began to cry and a  handsome man , and the  girl threw behind her a  glass which which the children  had laid her dead so  bed until the glass a  great glass of bread and  cut the great stones ,  but the king went home  and cut a great glass  one which for a glass  deal one pieces . the  old woman went home and  cut a great glass which  that it was a great  dove , and at last  one of joy which had  in a glass glass which  glass for a great glass  , and the girl was  over and cut a glass  its glass which a glass  joy , and the queen  again at each other and  saw a great deal which  for a great deal which  money in a glass which  pieces a glass , and  she went home and cut  her a glass which which  which a great deal one  money for a great glass  for a great deal which  times them a great glass  that a great deal which  money in a glass which  bread and long as the  girl looked so long that  she went home and cut  her a glass trees which  was a great glass which  wine and cut the glass  its glass and half that  that she was not so  beautiful that the children saw  all , and the girl  cut a great deal which  that the half which had  made a great glass which  that the glass was a  great glass , and the  little tailor , however ,  that she went to the  table , and cut her  hole which a great glass  as long as that a  wicked woman which was so  beautiful that the children saw  my human form . the  others , however , went  and cut a great glass  which that it was a  beautiful dove , and the  others saw that and was  all in a great glass  that the half which were  for a great glass ,  for the girl was so  beautiful that the girl saw  that a great glass of  wine and which he had  a great glass which were  half a great glass ,  until the girl threw a  great glass which that the  marriage which she went and  fetch her a great deal  that it , thought the  old woman , however ,  too that no one ever  said the whole over ,  and the girl said ,  how do you go and  fetch my axe , and  let it go and fetch   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 194)\n",
      "(62).(50).(60).(35).(91).(75).(81).(73).(79).(56).\n",
      "Average loss at step 195: 2.911898\n",
      "\tPerplexity at step 195: 18.391679\n",
      "\n",
      "Valid Perplexity: 82.89\n",
      "\n",
      "Generated Text after epoch 194 ... \n",
      "======================== New text Segment ==========================\n",
      " should do it for it ,  or i must not have  be my wife . the  girl , however , was  there for the girl ,  who went out and cut  the treasures its treasures which  such a glass such such  the stone that was the  queen , and that lay  in a glass trees which  it was not so long  for the old man ,  he went home and cut  his treasures and treasures and  half its treasures which which  had not come out of  it . the girl was  over the house , and  that was there , and  which went out and cut  it a strong treasures which  one horses began to cry  home and put it and  which it must carry him  for a stone which was  not so so as it  must have my wife like  such a stone and that  the girl was there a  little and which the child  came home and cut her  great pieces , which the  little tailor came home and  threw herself on her treasures  , laid him on her  table , laid them which  such the delicate ring which  had ever such an axe  . and now it must  have my bride , and  one speed , and have  hold it a treasures home  or be as long as  it will do you do  or do that i have  come out and glass and  which my wife have the  work my work that you  may do and you have  you be my wife .  and the child says ,  come out . the paddock  comes home , and the  son said , how do  i do what i have  like that . '' the  tailor went home and went  and went home and cut  the treasures which it should  do the axe . the  girl says , no ,  you have will be my  bride , and have no  rest for it that the  child , for which may  do you know , do  do my wife any one  anything , you must be  an bride , and the  son said , i will  have my wife have you  come and do you want  to do my wife wife  and do you want ,  you must do my human  work . you must do  your wife but the little  bird which are you for  it . the girl went  back and went out and  put it on the little  stone and went home and  took him its treasures and  wine that it was not  only as a little man  has such such a glass  such a glass that that  it must be my bride  . '' the girl went  out and cut her treasures  home and put an axe  home and see that the  girl had laid him on  the stone and which was  there in the kitchen ,  and which at last it  was not only so long  for the old woman ,  he went home and cut   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 195)\n",
      "(96).(91).(64).(76).(29).(69).(38).(21).(84).(85).\n",
      "Average loss at step 196: 3.009955\n",
      "\tPerplexity at step 196: 20.286485\n",
      "\n",
      "Valid Perplexity: 80.53\n",
      "\n",
      "\t Reducing learning rate\n",
      "Generated Text after epoch 195 ... \n",
      "======================== New text Segment ==========================\n",
      " had the king 's son and  the queen , and the  bride went home and went  out home and put the  wife in her bed and  asked her what the king  's daughter , and the  king 's son went home  again and said the king  , `` i will not  know that , '' said  the king and the king  's son went home and  told him that the king  , however , was so  beautiful that the king was  quite still to her ,  until the king 's son  went home and went home  and put it in the  dress , and told him  that the king 's son  was so beautiful that the  king was so beautiful that  the king 's son went  out and put it on  , and went home and  told him that the king  , however , was so  beautiful that the king was  sitting with a great lake  . if the king must  set him a great box  from their pieces until the  king 's daughter and the  son was so beautiful until  he went and put the  maiden in the kitchen ,  she went home and put  a great box which it  came and until the king  with the queen , and  the king 's son went  to pass , and that  went home and put the  dress in the air and  said that the king 's  bride , and the king  's son went home and  went home and put it  into the bed . the  girl said that the bride  would not know what the  queen would not know what  to be the bride ,  and the king 's son  went home and went home  and put it home and  cried , `` that is  not here you . ''  but the king 's son  led him and the son  at last the son was  still more beautiful until the  king 's son and the  king 's son went home  until the bride , and  opened the door and went  out home and went home  again , and the captain  said that the king which  had to pass until the  old woman was still more  beautiful dress which were all  so beautiful that it was  the bride , and the  bride cried , `` that  are the bride , and  what you can not have  my bride , and that  this the king 's son  were so beautiful that the  king must come home and  see that it must be  my bride . the king  's son went home and  kissed him and laid her  back in the bed ,  which were a feast in  her hand and cried that  that the king 's son  went home and went home  the king 's son and  all the castle was the  castle , and the king  , and the king 's  bride , and the king  , and the king 's  bride and went and put  it on and cried ,   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 196)\n",
      "(20).(44).(78).(11).(90).(84).(58).(15).(8).(50).\n",
      "Average loss at step 197: 2.909556\n",
      "\tPerplexity at step 197: 18.348647\n",
      "\n",
      "Valid Perplexity: 79.50\n",
      "\n",
      "Generated Text after epoch 196 ... \n",
      "======================== New text Segment ==========================\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " went into the forest and put  it on her little little  little table , and the  little ring which the king  who was so beautiful the  castle , she was so  beautiful that the king was  a feast , and went  home and cut her little  treasures and meat and laid  in the forest and the  king 's son went and  fetch her that the king  was so beautiful that the  king was so beautiful that  she could not hold her  that the king 's son  were so beautiful that she  did not know it ,  but the king said ,  no , you are my  only wife , and you  can be my wife and  you will not be so  much , you are my  son for the king and  no one could not be  hold that it . the  king was so beautiful that  it should be set out  with the table , and  went home and put it  and the table , and  that at last the king  looked on her hand and  and the king 's son  were so so that the  old man was so beautiful  a little dress , the  cook 's son , and  she must not hold the  glass which it . the  king 's son went home  with the king , however  which should have hold the  dress , but as soon  for the king 's son  also , the king 's  only wife that she did  the cry . the king  must come and pass and  that the king 's son  could not know how to  go and see that the  fox answered , you will  go home and fetch her  . the king said ,  no , you are my  wife to be so long  for the king , and  am not here to eat  and drink , but the  bride said that the king  was so beautiful that the  old man was so beautiful  a little dress , the  old king cried and the  cook , and the king  , who must not hold  his hair and put her  horse and cried the little  old man which was so  long as the king should  not hold it for her  the dress , but the  king said , i will  take it and my wife  my wife , but the  king said , i am  my bride to hold the  rest , and you have  an bride , and will  let her go home and  eat . the king 's  mother the king went home  home and the king 's  only wife the king 's  daughter , and she went  into the hall and put  his rest which had ever  become an beautiful ring which  which had ever set for  her wife and said ,  you are my bride ,  until you are my bride  but that you will be  my bride . and the  king was so beautiful that  it should be set out  for the king . the   \n",
      "====================================================================\n",
      "\n",
      "Training (Step: 197)\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-41-053ddefcf914>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     53\u001b[0m             \u001b[0mfeed_dict\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m{\u001b[0m\u001b[0mtf_learning_rate\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;36m0.0005\u001b[0m\u001b[1;33m}\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     54\u001b[0m             _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n\u001b[1;32m---> 55\u001b[1;33m                                                        feed_dict=feed_dict)\n\u001b[0m\u001b[0;32m     56\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     57\u001b[0m             \u001b[0maverage_loss\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mstep_perplexity\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    903\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    904\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 905\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    906\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    907\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1135\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1136\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1137\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1138\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1139\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1353\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1354\u001b[0m       return self._do_call(_run_fn, self._session, feeds, fetches, targets,\n\u001b[1;32m-> 1355\u001b[1;33m                            options, run_metadata)\n\u001b[0m\u001b[0;32m   1356\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1357\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1359\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1360\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1361\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1362\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1363\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(session, feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1338\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1339\u001b[0m           return tf_session.TF_Run(session, options, feed_dict, fetch_list,\n\u001b[1;32m-> 1340\u001b[1;33m                                    target_list, status, run_metadata)\n\u001b[0m\u001b[0;32m   1341\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1342\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0msession\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "beam_nodes = []\n",
    "\n",
    "train_perplexity_ot = []\n",
    "valid_perplexity_ot = []\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "print('Initialized')\n",
    "average_loss = 0\n",
    "\n",
    "# We use the first 10 documents that has \n",
    "# more than (num_steps+1)*steps_per_document bigrams for creating the validation dataset\n",
    "\n",
    "# Identify the first 10 documents following the above condition\n",
    "long_doc_ids = []\n",
    "for di in range(num_files):\n",
    "  if len(data_list[di])>1000:\n",
    "    long_doc_ids.append(di)\n",
    "  if len(long_doc_ids)==10:\n",
    "    break\n",
    "\n",
    "# Generating data\n",
    "data_gens = []\n",
    "valid_gens = []\n",
    "for fi in range(num_files):\n",
    "  # Get all the bigrams if the document id is not in the validation document ids\n",
    "  if fi not in long_doc_ids:\n",
    "    data_gens.append(DataGeneratorSeq(data_list[fi],batch_size,num_unrollings))\n",
    "  # if the document is in the validation doc ids, only get up to the \n",
    "  # last steps_per_document bigrams and use the last steps_per_document bigrams as validation data\n",
    "  else:\n",
    "    data_gens.append(DataGeneratorSeq(data_list[fi][:-steps_per_document],batch_size,num_unrollings))\n",
    "    # Defining the validation data generator\n",
    "    valid_gens.append(DataGeneratorSeq(data_list[fi][-steps_per_document:],1,1))\n",
    "\n",
    "valid_docs = len(valid_gens)\n",
    "assert valid_docs>0\n",
    "\n",
    "feed_dict = {}\n",
    "for step in range(num_steps):\n",
    "    print('Training (Step: %d)'%step)\n",
    "    for di in np.random.permutation(train_doc_count)[:docs_per_step]:            \n",
    "\n",
    "        for doc_step_id in range(steps_per_document):\n",
    "            \n",
    "            u_data, u_labels = data_gens[di].unroll_batches()\n",
    "            for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):            \n",
    "                feed_dict[train_inputs[ui]] = dat\n",
    "                feed_dict[train_labels[ui]] = lbl\n",
    "                #print(['( %s; %s ) '%(reverse_dictionary[tid],reverse_dictionary[til]) for tid,til in zip(np.argmax(dat,axis=1),np.argmax(lbl,axis=1))])\n",
    "            \n",
    "            feed_dict.update({tf_learning_rate:0.0005})\n",
    "            _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n",
    "                                                       feed_dict=feed_dict)\n",
    "            \n",
    "            average_loss += step_perplexity\n",
    "        \n",
    "        print('(%d).'%di,end='')\n",
    "    print('')    \n",
    "    \n",
    "    if (step+1) % valid_summary == 0:\n",
    "      \n",
    "      average_loss = average_loss / (docs_per_step*steps_per_document*valid_summary)\n",
    "      # The average loss is an estimate of the loss over the last 2000 batches.\n",
    "      print('Average loss at step %d: %f' % (step+1, average_loss))\n",
    "      print('\\tPerplexity at step %d: %f' %(step+1, np.exp(average_loss)))\n",
    "      train_perplexity_ot.append(np.exp(average_loss))\n",
    "      average_loss = 0 # reset loss\n",
    "      \n",
    "      valid_loss = 0 # reset loss\n",
    "        \n",
    "      # calculate valid perplexity\n",
    "      for v_doc_id in range(valid_docs):\n",
    "          # Remember we process things as bigrams\n",
    "          # So need to divide by 2\n",
    "          for v_step in range(steps_per_document//2):\n",
    "            uvalid_data,uvalid_labels = valid_gens[v_doc_id].unroll_batches()        \n",
    "\n",
    "            # Run validation phase related TensorFlow operations       \n",
    "            v_perp = session.run(\n",
    "                valid_perplexity_without_exp,\n",
    "                feed_dict = {valid_inputs:uvalid_data[0],valid_labels: uvalid_labels[0]}\n",
    "            )\n",
    "\n",
    "            valid_loss += v_perp\n",
    "            \n",
    "          # Reset validation data generator cursor\n",
    "          valid_gens[v_doc_id].reset_indices() \n",
    "      print()      \n",
    "      v_perplexity = np.exp(valid_loss/(steps_per_document*valid_docs//2))\n",
    "      print(\"Valid Perplexity: %.2f\\n\"%v_perplexity)\n",
    "      valid_perplexity_ot.append(v_perplexity)\n",
    "          \n",
    "      decay_learning_rate(session, v_perplexity)\n",
    "    \n",
    "      # Generating new text ...\n",
    "      # We will be generating one segment having 500 bigrams\n",
    "      # Feel free to generate several segments by changing\n",
    "      # the value of segments_to_generate\n",
    "      print('Generated Text after epoch %d ... '%step)  \n",
    "      segments_to_generate = 1\n",
    "      chars_in_segment = 500//beam_length\n",
    "    \n",
    "      for _ in range(segments_to_generate):\n",
    "        print('======================== New text Segment ==========================')\n",
    "        # first word randomly generated\n",
    "        test_word = data_list[np.random.randint(0,num_files)][np.random.randint(0,100)]\n",
    "        print(\"\",reverse_dictionary[test_word],end=' ')\n",
    "        \n",
    "        # Generating words within a segment with Beam Search\n",
    "        for _ in range(chars_in_segment):\n",
    "            test_sequence = get_beam_prediction(session)\n",
    "            print(test_sequence,end=' ')    \n",
    "        print(\" \")\n",
    "        session.run([reset_sample_beam_state])\n",
    "        \n",
    "        print('====================================================================')\n",
    "        \n",
    "      print(\"\")\n",
    "\n",
    "session.close()\n",
    "\n",
    "with open('lstm_beam_search_word2vec_rnn_api.csv', 'wt') as f:\n",
    "    writer = csv.writer(f,delimiter=',')\n",
    "    writer.writerow(train_perplexity_ot)\n",
    "    writer.writerow(valid_perplexity_ot)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
